diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index bd24eab3a..6074f14c6 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -17,176 +17,28 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: branch: 'bot/auto-update' - format: + create-matrix: runs-on: ubuntu-latest needs: create-branch + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Create Matrix + id: set-matrix + run: node scripts/create-matrix.js + format: + runs-on: ubuntu-latest + needs: create-matrix continue-on-error: true strategy: fail-fast: false - matrix: - country: - [ - ad, - ae, - af, - ag, - al, - am, - ao, - ar, - at, - au, - aw, - az, - ba, - bb, - bd, - be, - bf, - bg, - bh, - bn, - bo, - br, - bs, - by, - ca, - cd, - cg, - ch, - ci, - cl, - cm, - cn, - co, - cr, - cu, - cw, - cy, - cz, - de, - dk, - do, - dz, - ec, - ee, - eg, - es, - et, - fi, - fj, - fo, - fr, - pf, - ge, - gh, - gm, - gn, - gp, - gq, - gr, - gt, - hk, - hn, - hr, - ht, - hu, - id, - ie, - il, - in, - iq, - ir, - is, - it, - jm, - jo, - jp, - ke, - kg, - kh, - kp, - kr, - kw, - kz, - la, - lb, - li, - lk, - lt, - lu, - lv, - ly, - ma, - mc, - md, - me, - mk, - ml, - mm, - mn, - mo, - mt, - mv, - mx, - my, - mz, - ne, - ng, - ni, - nl, - no, - np, - nz, - om, - pa, - pe, - ph, - pk, - pl, - pr, - ps, - pt, - py, - qa, - ro, - rs, - ru, - rw, - sa, - sd, - se, - sg, - si, - sk, - sl, - sm, - sn, - so, - sv, - sy, - th, - tj, - tm, - tn, - tr, - tt, - tw, - tz, - ua, - ug, - uk, - us, - uy, - uz, - va, - ve, - vi, - vn, - xk, - ye, - zm - ] + matrix: ${{fromJSON(needs.create-matrix.outputs.matrix)}} steps: - name: Checkout uses: actions/checkout@v2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 40147d210..2d835b3e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -152,15 +152,14 @@ http://example.com/stream.m3u8 - `.github/` - `ISSUE_TEMPLATE/`: issue templates for this repository. - `workflows/` - - `auto-update.yml`: GitHub Action that automatically updates all playlists every day. - - `check.yml`: GitHub Action that automatically checks every pull request for syntax errors. + - `auto-update.yml`: GitHub workflow that automatically updates all playlists every day. + - `check.yml`: GitHub workflow that automatically checks every pull request for syntax errors. + - `cleanup.yml`: GitHub workflow that automatically removes broken links every week. - `CODE_OF_CONDUCT.md`: rules you shouldn't break if you don't want to get banned. - `.readme/` - - `_categories.md`: automatically generated list of all categories and their corresponding playlists. - - `_countries.md`: automatically generated list of all countries and their corresponding playlists. - - `_languages.md`: automatically generated list of all languages and their corresponding playlists. - `config.json`: config for the `markdown-include` package, which is used to compile everything into one `README.md` file. - `preview.png`: image displayed in the `README.md`. + - `supported-categories.md`: list of supported categories. - `supported-region-codes.md`: list of supported region codes. - `template.md`: template for `README.md`. - `channels/`: contains all channels broken down by the country from which they are broadcast. @@ -168,13 +167,14 @@ http://example.com/stream.m3u8 - `unsorted.m3u`: playlist with channels not yet sorted. - `scripts/` - `data/`: data used in scripts. - - `helpers/`: helper scripts used in GitHub Actions. - - `filter.js`: used within GitHub Action to remove blacklisted channels from playlists. - - `format.js`: used within GitHub Action to format channel descriptions. - - `generate.js`: used within GitHub Action to generate all additional playlists. - - `remove-duplicates.js`: used in GitHub Action to remove duplicates from the playlist. - - `sort.js`: used within GitHub Action to sort channels by name. - - `update-readme.js`: used within GitHub Action to update the `README.md` file. + - `helpers/`: helper scripts. + - `create-matrix.js`: used within GitHub workflow to create matrix of files to process. + - `filter.js`: used within GitHub workflow to remove blacklisted channels from playlists. + - `format.js`: used within GitHub workflow to format channel descriptions. + - `generate.js`: used within GitHub workflow to generate all additional playlists. + - `remove-broken-links.js`: used in GitHub workflow to remove broken links from the playlist. + - `remove-duplicates.js`: used in GitHub workflow to remove duplicates from the playlist. + - `sort.js`: used within GitHub workflow to sort channels by name. + - `update-readme.js`: used within GitHub workflow to update the `README.md` file. - `CONTRIBUTING.md`: file you are currently reading. -- `index.m3u`: main playlist that contains links to all playlists in the `channels/` folder. - `README.md`: project description generated from the contents of the `.readme/` folder. diff --git a/index.m3u b/index.m3u deleted file mode 100644 index 714c18671..000000000 --- a/index.m3u +++ /dev/null @@ -1,323 +0,0 @@ -#EXTM3U -#EXTINF:-1,Afghanistan -channels/af.m3u -#EXTINF:-1,Albania -channels/al.m3u -#EXTINF:-1,Algeria -channels/dz.m3u -#EXTINF:-1,Andorra -channels/ad.m3u -#EXTINF:-1,Angola -channels/ao.m3u -#EXTINF:-1,Antigua and Barbuda -channels/ag.m3u -#EXTINF:-1,Argentina -channels/ar.m3u -#EXTINF:-1,Armenia -channels/am.m3u -#EXTINF:-1,Aruba -channels/aw.m3u -#EXTINF:-1,Australia -channels/au.m3u -#EXTINF:-1,Austria -channels/at.m3u -#EXTINF:-1,Azerbaijan -channels/az.m3u -#EXTINF:-1,Bahamas -channels/bs.m3u -#EXTINF:-1,Bahrain -channels/bh.m3u -#EXTINF:-1,Bangladesh -channels/bd.m3u -#EXTINF:-1,Barbados -channels/bb.m3u -#EXTINF:-1,Belarus -channels/by.m3u -#EXTINF:-1,Belgium -channels/be.m3u -#EXTINF:-1,Bolivia -channels/bo.m3u -#EXTINF:-1,Bosnia and Herzegovina -channels/ba.m3u -#EXTINF:-1,Brazil -channels/br.m3u -#EXTINF:-1,Brunei -channels/bn.m3u -#EXTINF:-1,Bulgaria -channels/bg.m3u -#EXTINF:-1,Burkina Faso -channels/bf.m3u -#EXTINF:-1,Cambodia -channels/kh.m3u -#EXTINF:-1,Cameroon -channels/cm.m3u -#EXTINF:-1,Canada -channels/ca.m3u -#EXTINF:-1,Chile -channels/cl.m3u -#EXTINF:-1,China -channels/cn.m3u -#EXTINF:-1,Colombia -channels/co.m3u -#EXTINF:-1,Costa Rica -channels/cr.m3u -#EXTINF:-1,Croatia -channels/hr.m3u -#EXTINF:-1,Cuba -channels/cu.m3u -#EXTINF:-1,CuraƧao -channels/cw.m3u -#EXTINF:-1,Cyprus -channels/cy.m3u -#EXTINF:-1,Czech Republic -channels/cz.m3u -#EXTINF:-1,Democratic Republic of the Congo -channels/cd.m3u -#EXTINF:-1,Republic of the Congo -channels/cg.m3u -#EXTINF:-1,Denmark -channels/dk.m3u -#EXTINF:-1,Dominican Republic -channels/do.m3u -#EXTINF:-1,Ecuador -channels/ec.m3u -#EXTINF:-1,Egypt -channels/eg.m3u -#EXTINF:-1,El Salvador -channels/sv.m3u -#EXTINF:-1,Equatorial Guinea -channels/gq.m3u -#EXTINF:-1,Estonia -channels/ee.m3u -#EXTINF:-1,Ethiopia -channels/et.m3u -#EXTINF:-1,Faroe Islands -channels/fo.m3u -#EXTINF:-1,Finland -channels/fi.m3u -#EXTINF:-1,Fiji -channels/fj.m3u -#EXTINF:-1,France -channels/fr.m3u -#EXTINF:-1,French Polynesia -channels/pf.m3u -#EXTINF:-1,Gambia -channels/gm.m3u -#EXTINF:-1,Georgia -channels/ge.m3u -#EXTINF:-1,Germany -channels/de.m3u -#EXTINF:-1,Ghana -channels/gh.m3u -#EXTINF:-1,Greece -channels/gr.m3u -#EXTINF:-1,Guadeloupe -channels/gp.m3u -#EXTINF:-1,Guatemala -channels/gt.m3u -#EXTINF:-1,Guinea -channels/gn.m3u -#EXTINF:-1,Haiti -channels/ht.m3u -#EXTINF:-1,Honduras -channels/hn.m3u -#EXTINF:-1,Hong Kong -channels/hk.m3u -#EXTINF:-1,Hungary -channels/hu.m3u -#EXTINF:-1,Iceland -channels/is.m3u -#EXTINF:-1,India -channels/in.m3u -#EXTINF:-1,Indonesia -channels/id.m3u -#EXTINF:-1,Iran -channels/ir.m3u -#EXTINF:-1,Iraq -channels/iq.m3u -#EXTINF:-1,Ireland -channels/ie.m3u -#EXTINF:-1,Israel -channels/il.m3u -#EXTINF:-1,Italy -channels/it.m3u -#EXTINF:-1,Ivory Coast -channels/ci.m3u -#EXTINF:-1,Jamaica -channels/jm.m3u -#EXTINF:-1,Japan -channels/jp.m3u -#EXTINF:-1,Jordan -channels/jo.m3u -#EXTINF:-1,Kazakhstan -channels/kz.m3u -#EXTINF:-1,Kenya -channels/ke.m3u -#EXTINF:-1,Kosovo -channels/xk.m3u -#EXTINF:-1,Kuwait -channels/kw.m3u -#EXTINF:-1,Kyrgyzstan -channels/kg.m3u -#EXTINF:-1,Laos -channels/la.m3u -#EXTINF:-1,Latvia -channels/lv.m3u -#EXTINF:-1,Lebanon -channels/lb.m3u -#EXTINF:-1,Libya -channels/ly.m3u -#EXTINF:-1,Liechtenstein -channels/li.m3u -#EXTINF:-1,Lithuania -channels/lt.m3u -#EXTINF:-1,Luxembourg -channels/lu.m3u -#EXTINF:-1,Macau -channels/mo.m3u -#EXTINF:-1,Malaysia -channels/my.m3u -#EXTINF:-1,Maldives -channels/mv.m3u -#EXTINF:-1,Mali -channels/ml.m3u -#EXTINF:-1,Malta -channels/mt.m3u -#EXTINF:-1,Mexico -channels/mx.m3u -#EXTINF:-1,Moldova -channels/md.m3u -#EXTINF:-1,Monaco -channels/mc.m3u -#EXTINF:-1,Mongolia -channels/mn.m3u -#EXTINF:-1,Montenegro -channels/me.m3u -#EXTINF:-1,Morocco -channels/ma.m3u -#EXTINF:-1,Mozambique -channels/mz.m3u -#EXTINF:-1,Myanmar -channels/mm.m3u -#EXTINF:-1,Nepal -channels/np.m3u -#EXTINF:-1,Netherlands -channels/nl.m3u -#EXTINF:-1,New Zealand -channels/nz.m3u -#EXTINF:-1,Nicaragua -channels/ni.m3u -#EXTINF:-1,Niger -channels/ne.m3u -#EXTINF:-1,Nigeria -channels/ng.m3u -#EXTINF:-1,North Korea -channels/kp.m3u -#EXTINF:-1,North Macedonia -channels/mk.m3u -#EXTINF:-1,Norway -channels/no.m3u -#EXTINF:-1,Oman -channels/om.m3u -#EXTINF:-1,Pakistan -channels/pk.m3u -#EXTINF:-1,Palestine -channels/ps.m3u -#EXTINF:-1,Panama -channels/pa.m3u -#EXTINF:-1,Paraguay -channels/py.m3u -#EXTINF:-1,Peru -channels/pe.m3u -#EXTINF:-1,Philippines -channels/ph.m3u -#EXTINF:-1,Poland -channels/pl.m3u -#EXTINF:-1,Portugal -channels/pt.m3u -#EXTINF:-1,Puerto Rico -channels/pr.m3u -#EXTINF:-1,Qatar -channels/qa.m3u -#EXTINF:-1,Romania -channels/ro.m3u -#EXTINF:-1,Russia -channels/ru.m3u -#EXTINF:-1,Rwanda -channels/rw.m3u -#EXTINF:-1,San Marino -channels/sm.m3u -#EXTINF:-1,Saudi Arabia -channels/sa.m3u -#EXTINF:-1,Senegal -channels/sn.m3u -#EXTINF:-1,Serbia -channels/rs.m3u -#EXTINF:-1,Sierra Leone -channels/sl.m3u -#EXTINF:-1,Singapore -channels/sg.m3u -#EXTINF:-1,Slovakia -channels/sk.m3u -#EXTINF:-1,Slovenia -channels/si.m3u -#EXTINF:-1,Somalia -channels/so.m3u -#EXTINF:-1,South Korea -channels/kr.m3u -#EXTINF:-1,Spain -channels/es.m3u -#EXTINF:-1,Sri Lanka -channels/lk.m3u -#EXTINF:-1,Sudan -channels/sd.m3u -#EXTINF:-1,Sweden -channels/se.m3u -#EXTINF:-1,Switzerland -channels/ch.m3u -#EXTINF:-1,Syria -channels/sy.m3u -#EXTINF:-1,Taiwan -channels/tw.m3u -#EXTINF:-1,Tajikistan -channels/tj.m3u -#EXTINF:-1,Tanzania -channels/tz.m3u -#EXTINF:-1,Thailand -channels/th.m3u -#EXTINF:-1,Trinidad and Tobago -channels/tt.m3u -#EXTINF:-1,Tunisia -channels/tn.m3u -#EXTINF:-1,Turkey -channels/tr.m3u -#EXTINF:-1,Turkmenistan -channels/tm.m3u -#EXTINF:-1,Uganda -channels/ug.m3u -#EXTINF:-1,Ukraine -channels/ua.m3u -#EXTINF:-1,United Arab Emirates -channels/ae.m3u -#EXTINF:-1,United Kingdom -channels/uk.m3u -#EXTINF:-1,United States -channels/us.m3u -#EXTINF:-1,Uruguay -channels/uy.m3u -#EXTINF:-1,Uzbekistan -channels/uz.m3u -#EXTINF:-1,Vatican City -channels/va.m3u -#EXTINF:-1,Venezuela -channels/ve.m3u -#EXTINF:-1,Vietnam -channels/vn.m3u -#EXTINF:-1,Virgin Islands of the United States -channels/vi.m3u -#EXTINF:-1,Yemen -channels/ye.m3u -#EXTINF:-1,Zambia -channels/zm.m3u -#EXTINF:-1,Unsorted -channels/unsorted.m3u diff --git a/scripts/create-matrix.js b/scripts/create-matrix.js new file mode 100644 index 000000000..6e306d42b --- /dev/null +++ b/scripts/create-matrix.js @@ -0,0 +1,9 @@ +const file = require('./helpers/file') + +file.list().then(files => { + files = files.filter(file => file !== 'channels/unsorted.m3u') + const country = files.map(file => file.replace(/channels\/|\.m3u/gi, '')) + const matrix = { country } + const output = `::set-output name=matrix::${JSON.stringify(matrix)}` + console.log(output) +}) diff --git a/scripts/filter.js b/scripts/filter.js index 3efbc492e..f6676c937 100644 --- a/scripts/filter.js +++ b/scripts/filter.js @@ -1,16 +1,17 @@ const blacklist = require('./data/blacklist.json') const parser = require('./helpers/parser') +const file = require('./helpers/file') const log = require('./helpers/log') async function main() { log.start() - log.print(`Parsing 'index.m3u'...`) - const playlists = parser.parseIndex() - for (const playlist of playlists) { - log.print(`\nProcessing '${playlist.url}'...`) + const files = await file.list() + if (!files.length) log.print(`No files is selected\n`) + for (const file of files) { + log.print(`\nProcessing '${file}'...`) await parser - .parsePlaylist(playlist.url) + .parsePlaylist(file) .then(removeBlacklisted) .then(p => p.save()) } diff --git a/scripts/format.js b/scripts/format.js index f1060295d..f70d11b39 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -27,11 +27,12 @@ let buffer, origins async function main() { log.start() - let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') - playlists = utils.filterPlaylists(playlists, config.country, config.exclude) - if (!playlists.length) log.print(`No playlist is selected\n`) - for (const playlist of playlists) { - await parser.parsePlaylist(playlist.url).then(updatePlaylist).then(savePlaylist) + const include = config.country.split(',').filter(i => i) + const exclude = config.exclude.split(',').filter(i => i) + let files = await file.list(include, exclude) + if (!files.length) log.print(`No files is selected\n`) + for (const file of files) { + await parser.parsePlaylist(file).then(updatePlaylist).then(savePlaylist) } log.finish() diff --git a/scripts/helpers/db.js b/scripts/helpers/db.js index 3354071c3..859a41576 100644 --- a/scripts/helpers/db.js +++ b/scripts/helpers/db.js @@ -1,13 +1,14 @@ const categories = require('../data/categories') const parser = require('./parser') const utils = require('./utils') +const file = require('./file') const db = {} db.load = async function () { - const items = parser.parseIndex() - for (const item of items) { - const playlist = await parser.parsePlaylist(item.url) + let files = await file.list() + for (const file of files) { + const playlist = await parser.parsePlaylist(file) db.playlists.add(playlist) for (const channel of playlist.channels) { db.channels.add(channel) diff --git a/scripts/helpers/file.js b/scripts/helpers/file.js index 578c76a73..44ff35135 100644 --- a/scripts/helpers/file.js +++ b/scripts/helpers/file.js @@ -1,10 +1,29 @@ const markdownInclude = require('markdown-include') const path = require('path') +const glob = require('glob') const fs = require('fs') const rootPath = path.resolve(__dirname) + '/../../' const file = {} +file.list = function (include = [], exclude = []) { + return new Promise(resolve => { + glob('channels/**/*.m3u', function (err, files) { + if (include.length) { + include = include.map(filename => `channels/${filename}.m3u`) + files = files.filter(filename => include.includes(filename)) + } + + if (exclude.length) { + exclude = exclude.map(filename => `channels/${filename}.m3u`) + files = files.filter(filename => !exclude.includes(filename)) + } + + resolve(files) + }) + }) +} + file.getFilename = function (filename) { return path.parse(filename).name } diff --git a/scripts/helpers/parser.js b/scripts/helpers/parser.js index 98f315c16..138af5907 100644 --- a/scripts/helpers/parser.js +++ b/scripts/helpers/parser.js @@ -5,13 +5,6 @@ const file = require('./file') const parser = {} -parser.parseIndex = function () { - const content = file.read('index.m3u') - const result = playlistParser.parse(content) - - return result.items -} - parser.parsePlaylist = async function (url) { const content = file.read(url) const result = playlistParser.parse(content) diff --git a/scripts/helpers/utils.js b/scripts/helpers/utils.js index 277bae92f..eb8f3d814 100644 --- a/scripts/helpers/utils.js +++ b/scripts/helpers/utils.js @@ -74,22 +74,6 @@ utils.removeProtocol = function (string) { return string.replace(/(^\w+:|^)\/\//, '') } -utils.filterPlaylists = function (arr, include = '', exclude = '') { - if (include) { - const included = include.split(',').map(filename => `channels/${filename}.m3u`) - - return arr.filter(i => included.indexOf(i.url) > -1) - } - - if (exclude) { - const excluded = exclude.split(',').map(filename => `channels/${filename}.m3u`) - - return arr.filter(i => excluded.indexOf(i.url) === -1) - } - - return arr -} - utils.sleep = function (ms) { return function (x) { return new Promise(resolve => setTimeout(() => resolve(x), ms)) diff --git a/scripts/remove-broken-links.js b/scripts/remove-broken-links.js index 3faf9968b..a6fff7a85 100644 --- a/scripts/remove-broken-links.js +++ b/scripts/remove-broken-links.js @@ -1,15 +1,17 @@ const parser = require('./helpers/parser') +const file = require('./helpers/file') const log = require('./helpers/log') async function main() { log.start() - log.print(`Parsing 'index.m3u'...`) - const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') - for (const playlist of playlists) { - log.print(`\nProcessing '${playlist.url}'...`) + let files = await file.list() + if (!files.length) log.print(`No files is selected\n`) + files = files.filter(file => file !== 'channels/unsorted.m3u') + for (const file of files) { + log.print(`\nProcessing '${file}'...`) await parser - .parsePlaylist(playlist.url) + .parsePlaylist(file) .then(removeBrokenLinks) .then(p => p.save()) } diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js index aa2c3d827..c75882f70 100644 --- a/scripts/remove-duplicates.js +++ b/scripts/remove-duplicates.js @@ -1,5 +1,6 @@ const parser = require('./helpers/parser') const utils = require('./helpers/utils') +const file = require('./helpers/file') const log = require('./helpers/log') let globalBuffer = [] @@ -7,18 +8,19 @@ let globalBuffer = [] async function main() { log.start() - log.print(`Parsing 'index.m3u'...`) - const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') - for (const playlist of playlists) { - log.print(`\nProcessing '${playlist.url}'...`) + let files = await file.list() + if (!files.length) log.print(`No files is selected\n`) + files = files.filter(file => file !== 'channels/unsorted.m3u') + for (const file of files) { + log.print(`\nProcessing '${file}'...`) await parser - .parsePlaylist(playlist.url) + .parsePlaylist(file) .then(addToGlobalBuffer) .then(removeDuplicates) .then(p => p.save()) } - if (playlists.length) { + if (files.length) { log.print(`\nProcessing 'channels/unsorted.m3u'...`) await parser .parsePlaylist('channels/unsorted.m3u') diff --git a/scripts/sort.js b/scripts/sort.js index 17560b848..976dfcae0 100644 --- a/scripts/sort.js +++ b/scripts/sort.js @@ -1,16 +1,18 @@ const parser = require('./helpers/parser') const utils = require('./helpers/utils') +const file = require('./helpers/file') const log = require('./helpers/log') async function main() { log.start() - log.print(`Parsing 'index.m3u'...`) - let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') - for (const playlist of playlists) { - log.print(`\nProcessing '${playlist.url}'...`) + let files = await file.list() + if (!files.length) log.print(`No files is selected\n`) + files = files.filter(file => file !== 'channels/unsorted.m3u') + for (const file of files) { + log.print(`\nProcessing '${file}'...`) await parser - .parsePlaylist(playlist.url) + .parsePlaylist(file) .then(sortChannels) .then(p => p.save()) }