Merge pull request #4425 from iptv-org/delete-index-m3u

Delete index.m3u

.github/workflows/auto-update.yml

@@ -17,176 +17,28 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           branch: 'bot/auto-update'
-  format:
+  create-matrix:
     runs-on: ubuntu-latest
     needs: create-branch
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: bot/auto-update
+      - name: Install Dependencies
+        run: npm install
+      - name: Create Matrix
+        id: set-matrix
+        run: node scripts/create-matrix.js
+  format:
+    runs-on: ubuntu-latest
+    needs: create-matrix
     continue-on-error: true
     strategy:
       fail-fast: false
-      matrix:
-        country:
-          [
-            ad,
-            ae,
-            af,
-            ag,
-            al,
-            am,
-            ao,
-            ar,
-            at,
-            au,
-            aw,
-            az,
-            ba,
-            bb,
-            bd,
-            be,
-            bf,
-            bg,
-            bh,
-            bn,
-            bo,
-            br,
-            bs,
-            by,
-            ca,
-            cd,
-            cg,
-            ch,
-            ci,
-            cl,
-            cm,
-            cn,
-            co,
-            cr,
-            cu,
-            cw,
-            cy,
-            cz,
-            de,
-            dk,
-            do,
-            dz,
-            ec,
-            ee,
-            eg,
-            es,
-            et,
-            fi,
-            fj,
-            fo,
-            fr,
-            pf,
-            ge,
-            gh,
-            gm,
-            gn,
-            gp,
-            gq,
-            gr,
-            gt,
-            hk,
-            hn,
-            hr,
-            ht,
-            hu,
-            id,
-            ie,
-            il,
-            in,
-            iq,
-            ir,
-            is,
-            it,
-            jm,
-            jo,
-            jp,
-            ke,
-            kg,
-            kh,
-            kp,
-            kr,
-            kw,
-            kz,
-            la,
-            lb,
-            li,
-            lk,
-            lt,
-            lu,
-            lv,
-            ly,
-            ma,
-            mc,
-            md,
-            me,
-            mk,
-            ml,
-            mm,
-            mn,
-            mo,
-            mt,
-            mv,
-            mx,
-            my,
-            mz,
-            ne,
-            ng,
-            ni,
-            nl,
-            no,
-            np,
-            nz,
-            om,
-            pa,
-            pe,
-            ph,
-            pk,
-            pl,
-            pr,
-            ps,
-            pt,
-            py,
-            qa,
-            ro,
-            rs,
-            ru,
-            rw,
-            sa,
-            sd,
-            se,
-            sg,
-            si,
-            sk,
-            sl,
-            sm,
-            sn,
-            so,
-            sv,
-            sy,
-            th,
-            tj,
-            tm,
-            tn,
-            tr,
-            tt,
-            tw,
-            tz,
-            ua,
-            ug,
-            uk,
-            us,
-            uy,
-            uz,
-            va,
-            ve,
-            vi,
-            vn,
-            xk,
-            ye,
-            zm
-          ]
+      matrix: ${{fromJSON(needs.create-matrix.outputs.matrix)}}
     steps:
       - name: Checkout
         uses: actions/checkout@v2

CONTRIBUTING.md

@@ -152,15 +152,14 @@ http://example.com/stream.m3u8
 - `.github/`
   - `ISSUE_TEMPLATE/`: issue templates for this repository.
   - `workflows/`
-    - `auto-update.yml`: GitHub Action that automatically updates all playlists every day.
-    - `check.yml`: GitHub Action that automatically checks every pull request for syntax errors.
+    - `auto-update.yml`: GitHub workflow that automatically updates all playlists every day.
+    - `check.yml`: GitHub workflow that automatically checks every pull request for syntax errors.
+    - `cleanup.yml`: GitHub workflow that automatically removes broken links every week.
   - `CODE_OF_CONDUCT.md`: rules you shouldn't break if you don't want to get banned.
 - `.readme/`
-  - `_categories.md`: automatically generated list of all categories and their corresponding playlists.
-  - `_countries.md`: automatically generated list of all countries and their corresponding playlists.
-  - `_languages.md`: automatically generated list of all languages and their corresponding playlists.
   - `config.json`: config for the `markdown-include` package, which is used to compile everything into one `README.md` file.
   - `preview.png`: image displayed in the `README.md`.
+  - `supported-categories.md`: list of supported categories.
   - `supported-region-codes.md`: list of supported region codes.
   - `template.md`: template for `README.md`.
 - `channels/`: contains all channels broken down by the country from which they are broadcast.
@@ -168,13 +167,14 @@ http://example.com/stream.m3u8
   - `unsorted.m3u`: playlist with channels not yet sorted.
 - `scripts/`
   - `data/`: data used in scripts.
-  - `helpers/`: helper scripts used in GitHub Actions.
-  - `filter.js`: used within GitHub Action to remove blacklisted channels from playlists.
-  - `format.js`: used within GitHub Action to format channel descriptions.
-  - `generate.js`: used within GitHub Action to generate all additional playlists.
-  - `remove-duplicates.js`: used in GitHub Action to remove duplicates from the playlist.
-  - `sort.js`: used within GitHub Action to sort channels by name.
-  - `update-readme.js`: used within GitHub Action to update the `README.md` file.
+  - `helpers/`: helper scripts.
+  - `create-matrix.js`: used within GitHub workflow to create matrix of files to process.
+  - `filter.js`: used within GitHub workflow to remove blacklisted channels from playlists.
+  - `format.js`: used within GitHub workflow to format channel descriptions.
+  - `generate.js`: used within GitHub workflow to generate all additional playlists.
+  - `remove-broken-links.js`: used in GitHub workflow to remove broken links from the playlist.
+  - `remove-duplicates.js`: used in GitHub workflow to remove duplicates from the playlist.
+  - `sort.js`: used within GitHub workflow to sort channels by name.
+  - `update-readme.js`: used within GitHub workflow to update the `README.md` file.
 - `CONTRIBUTING.md`: file you are currently reading.
-- `index.m3u`: main playlist that contains links to all playlists in the `channels/` folder.
 - `README.md`: project description generated from the contents of the `.readme/` folder.

index.m3u

@@ -1,323 +0,0 @@
-#EXTM3U 
-#EXTINF:-1,Afghanistan
-channels/af.m3u
-#EXTINF:-1,Albania
-channels/al.m3u
-#EXTINF:-1,Algeria
-channels/dz.m3u
-#EXTINF:-1,Andorra
-channels/ad.m3u
-#EXTINF:-1,Angola
-channels/ao.m3u
-#EXTINF:-1,Antigua and Barbuda
-channels/ag.m3u
-#EXTINF:-1,Argentina
-channels/ar.m3u
-#EXTINF:-1,Armenia
-channels/am.m3u
-#EXTINF:-1,Aruba
-channels/aw.m3u
-#EXTINF:-1,Australia
-channels/au.m3u
-#EXTINF:-1,Austria
-channels/at.m3u
-#EXTINF:-1,Azerbaijan
-channels/az.m3u
-#EXTINF:-1,Bahamas
-channels/bs.m3u
-#EXTINF:-1,Bahrain
-channels/bh.m3u
-#EXTINF:-1,Bangladesh
-channels/bd.m3u
-#EXTINF:-1,Barbados
-channels/bb.m3u
-#EXTINF:-1,Belarus
-channels/by.m3u
-#EXTINF:-1,Belgium
-channels/be.m3u
-#EXTINF:-1,Bolivia
-channels/bo.m3u
-#EXTINF:-1,Bosnia and Herzegovina
-channels/ba.m3u
-#EXTINF:-1,Brazil
-channels/br.m3u
-#EXTINF:-1,Brunei
-channels/bn.m3u
-#EXTINF:-1,Bulgaria
-channels/bg.m3u
-#EXTINF:-1,Burkina Faso
-channels/bf.m3u
-#EXTINF:-1,Cambodia
-channels/kh.m3u
-#EXTINF:-1,Cameroon
-channels/cm.m3u
-#EXTINF:-1,Canada
-channels/ca.m3u
-#EXTINF:-1,Chile
-channels/cl.m3u
-#EXTINF:-1,China
-channels/cn.m3u
-#EXTINF:-1,Colombia
-channels/co.m3u
-#EXTINF:-1,Costa Rica
-channels/cr.m3u
-#EXTINF:-1,Croatia
-channels/hr.m3u
-#EXTINF:-1,Cuba
-channels/cu.m3u
-#EXTINF:-1,Curaçao
-channels/cw.m3u
-#EXTINF:-1,Cyprus
-channels/cy.m3u
-#EXTINF:-1,Czech Republic
-channels/cz.m3u
-#EXTINF:-1,Democratic Republic of the Congo
-channels/cd.m3u
-#EXTINF:-1,Republic of the Congo
-channels/cg.m3u
-#EXTINF:-1,Denmark
-channels/dk.m3u
-#EXTINF:-1,Dominican Republic
-channels/do.m3u
-#EXTINF:-1,Ecuador
-channels/ec.m3u
-#EXTINF:-1,Egypt
-channels/eg.m3u
-#EXTINF:-1,El Salvador
-channels/sv.m3u
-#EXTINF:-1,Equatorial Guinea
-channels/gq.m3u
-#EXTINF:-1,Estonia
-channels/ee.m3u
-#EXTINF:-1,Ethiopia
-channels/et.m3u
-#EXTINF:-1,Faroe Islands
-channels/fo.m3u
-#EXTINF:-1,Finland
-channels/fi.m3u
-#EXTINF:-1,Fiji
-channels/fj.m3u
-#EXTINF:-1,France
-channels/fr.m3u
-#EXTINF:-1,French Polynesia
-channels/pf.m3u
-#EXTINF:-1,Gambia
-channels/gm.m3u
-#EXTINF:-1,Georgia
-channels/ge.m3u
-#EXTINF:-1,Germany
-channels/de.m3u
-#EXTINF:-1,Ghana
-channels/gh.m3u
-#EXTINF:-1,Greece
-channels/gr.m3u
-#EXTINF:-1,Guadeloupe
-channels/gp.m3u
-#EXTINF:-1,Guatemala
-channels/gt.m3u
-#EXTINF:-1,Guinea
-channels/gn.m3u
-#EXTINF:-1,Haiti
-channels/ht.m3u
-#EXTINF:-1,Honduras
-channels/hn.m3u
-#EXTINF:-1,Hong Kong
-channels/hk.m3u
-#EXTINF:-1,Hungary
-channels/hu.m3u
-#EXTINF:-1,Iceland
-channels/is.m3u
-#EXTINF:-1,India
-channels/in.m3u
-#EXTINF:-1,Indonesia
-channels/id.m3u
-#EXTINF:-1,Iran
-channels/ir.m3u
-#EXTINF:-1,Iraq
-channels/iq.m3u
-#EXTINF:-1,Ireland
-channels/ie.m3u
-#EXTINF:-1,Israel
-channels/il.m3u
-#EXTINF:-1,Italy
-channels/it.m3u
-#EXTINF:-1,Ivory Coast
-channels/ci.m3u
-#EXTINF:-1,Jamaica
-channels/jm.m3u
-#EXTINF:-1,Japan
-channels/jp.m3u
-#EXTINF:-1,Jordan
-channels/jo.m3u
-#EXTINF:-1,Kazakhstan
-channels/kz.m3u
-#EXTINF:-1,Kenya
-channels/ke.m3u
-#EXTINF:-1,Kosovo
-channels/xk.m3u
-#EXTINF:-1,Kuwait
-channels/kw.m3u
-#EXTINF:-1,Kyrgyzstan
-channels/kg.m3u
-#EXTINF:-1,Laos
-channels/la.m3u
-#EXTINF:-1,Latvia
-channels/lv.m3u
-#EXTINF:-1,Lebanon
-channels/lb.m3u
-#EXTINF:-1,Libya
-channels/ly.m3u
-#EXTINF:-1,Liechtenstein
-channels/li.m3u
-#EXTINF:-1,Lithuania
-channels/lt.m3u
-#EXTINF:-1,Luxembourg
-channels/lu.m3u
-#EXTINF:-1,Macau
-channels/mo.m3u
-#EXTINF:-1,Malaysia
-channels/my.m3u
-#EXTINF:-1,Maldives
-channels/mv.m3u
-#EXTINF:-1,Mali
-channels/ml.m3u
-#EXTINF:-1,Malta
-channels/mt.m3u
-#EXTINF:-1,Mexico
-channels/mx.m3u
-#EXTINF:-1,Moldova
-channels/md.m3u
-#EXTINF:-1,Monaco
-channels/mc.m3u
-#EXTINF:-1,Mongolia
-channels/mn.m3u
-#EXTINF:-1,Montenegro
-channels/me.m3u
-#EXTINF:-1,Morocco
-channels/ma.m3u
-#EXTINF:-1,Mozambique
-channels/mz.m3u
-#EXTINF:-1,Myanmar
-channels/mm.m3u
-#EXTINF:-1,Nepal
-channels/np.m3u
-#EXTINF:-1,Netherlands
-channels/nl.m3u
-#EXTINF:-1,New Zealand
-channels/nz.m3u
-#EXTINF:-1,Nicaragua
-channels/ni.m3u
-#EXTINF:-1,Niger
-channels/ne.m3u
-#EXTINF:-1,Nigeria
-channels/ng.m3u
-#EXTINF:-1,North Korea
-channels/kp.m3u
-#EXTINF:-1,North Macedonia
-channels/mk.m3u
-#EXTINF:-1,Norway
-channels/no.m3u
-#EXTINF:-1,Oman
-channels/om.m3u
-#EXTINF:-1,Pakistan
-channels/pk.m3u
-#EXTINF:-1,Palestine
-channels/ps.m3u
-#EXTINF:-1,Panama
-channels/pa.m3u
-#EXTINF:-1,Paraguay
-channels/py.m3u
-#EXTINF:-1,Peru
-channels/pe.m3u
-#EXTINF:-1,Philippines
-channels/ph.m3u
-#EXTINF:-1,Poland
-channels/pl.m3u
-#EXTINF:-1,Portugal
-channels/pt.m3u
-#EXTINF:-1,Puerto Rico
-channels/pr.m3u
-#EXTINF:-1,Qatar
-channels/qa.m3u
-#EXTINF:-1,Romania
-channels/ro.m3u
-#EXTINF:-1,Russia
-channels/ru.m3u
-#EXTINF:-1,Rwanda
-channels/rw.m3u
-#EXTINF:-1,San Marino
-channels/sm.m3u
-#EXTINF:-1,Saudi Arabia
-channels/sa.m3u
-#EXTINF:-1,Senegal
-channels/sn.m3u
-#EXTINF:-1,Serbia
-channels/rs.m3u
-#EXTINF:-1,Sierra Leone
-channels/sl.m3u
-#EXTINF:-1,Singapore
-channels/sg.m3u
-#EXTINF:-1,Slovakia
-channels/sk.m3u
-#EXTINF:-1,Slovenia
-channels/si.m3u
-#EXTINF:-1,Somalia
-channels/so.m3u
-#EXTINF:-1,South Korea
-channels/kr.m3u
-#EXTINF:-1,Spain
-channels/es.m3u
-#EXTINF:-1,Sri Lanka
-channels/lk.m3u
-#EXTINF:-1,Sudan
-channels/sd.m3u
-#EXTINF:-1,Sweden
-channels/se.m3u
-#EXTINF:-1,Switzerland
-channels/ch.m3u
-#EXTINF:-1,Syria
-channels/sy.m3u
-#EXTINF:-1,Taiwan
-channels/tw.m3u
-#EXTINF:-1,Tajikistan
-channels/tj.m3u
-#EXTINF:-1,Tanzania
-channels/tz.m3u
-#EXTINF:-1,Thailand
-channels/th.m3u
-#EXTINF:-1,Trinidad and Tobago
-channels/tt.m3u
-#EXTINF:-1,Tunisia
-channels/tn.m3u
-#EXTINF:-1,Turkey
-channels/tr.m3u
-#EXTINF:-1,Turkmenistan
-channels/tm.m3u
-#EXTINF:-1,Uganda
-channels/ug.m3u
-#EXTINF:-1,Ukraine
-channels/ua.m3u
-#EXTINF:-1,United Arab Emirates
-channels/ae.m3u
-#EXTINF:-1,United Kingdom
-channels/uk.m3u
-#EXTINF:-1,United States
-channels/us.m3u
-#EXTINF:-1,Uruguay
-channels/uy.m3u
-#EXTINF:-1,Uzbekistan
-channels/uz.m3u
-#EXTINF:-1,Vatican City
-channels/va.m3u
-#EXTINF:-1,Venezuela
-channels/ve.m3u
-#EXTINF:-1,Vietnam
-channels/vn.m3u
-#EXTINF:-1,Virgin Islands of the United States
-channels/vi.m3u
-#EXTINF:-1,Yemen
-channels/ye.m3u
-#EXTINF:-1,Zambia
-channels/zm.m3u
-#EXTINF:-1,Unsorted
-channels/unsorted.m3u

scripts/create-matrix.js

@@ -0,0 +1,9 @@
+const file = require('./helpers/file')
+
+file.list().then(files => {
+  files = files.filter(file => file !== 'channels/unsorted.m3u')
+  const country = files.map(file => file.replace(/channels\/|\.m3u/gi, ''))
+  const matrix = { country }
+  const output = `::set-output name=matrix::${JSON.stringify(matrix)}`
+  console.log(output)
+})

scripts/filter.js

@@ -1,16 +1,17 @@
 const blacklist = require('./data/blacklist.json')
 const parser = require('./helpers/parser')
+const file = require('./helpers/file')
 const log = require('./helpers/log')
 
 async function main() {
   log.start()
 
-  log.print(`Parsing 'index.m3u'...`)
-  const playlists = parser.parseIndex()
-  for (const playlist of playlists) {
-    log.print(`\nProcessing '${playlist.url}'...`)
+  const files = await file.list()
+  if (!files.length) log.print(`No files is selected\n`)
+  for (const file of files) {
+    log.print(`\nProcessing '${file}'...`)
     await parser
-      .parsePlaylist(playlist.url)
+      .parsePlaylist(file)
       .then(removeBlacklisted)
       .then(p => p.save())
   }

scripts/format.js

@@ -27,11 +27,12 @@ let buffer, origins
 async function main() {
   log.start()
 
-  let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
-  playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
-  if (!playlists.length) log.print(`No playlist is selected\n`)
-  for (const playlist of playlists) {
-    await parser.parsePlaylist(playlist.url).then(updatePlaylist).then(savePlaylist)
+  const include = config.country.split(',').filter(i => i)
+  const exclude = config.exclude.split(',').filter(i => i)
+  let files = await file.list(include, exclude)
+  if (!files.length) log.print(`No files is selected\n`)
+  for (const file of files) {
+    await parser.parsePlaylist(file).then(updatePlaylist).then(savePlaylist)
   }
 
   log.finish()

scripts/helpers/db.js

@@ -1,13 +1,14 @@
 const categories = require('../data/categories')
 const parser = require('./parser')
 const utils = require('./utils')
+const file = require('./file')
 
 const db = {}
 
 db.load = async function () {
-  const items = parser.parseIndex()
-  for (const item of items) {
-    const playlist = await parser.parsePlaylist(item.url)
+  let files = await file.list()
+  for (const file of files) {
+    const playlist = await parser.parsePlaylist(file)
     db.playlists.add(playlist)
     for (const channel of playlist.channels) {
       db.channels.add(channel)

scripts/helpers/file.js

@@ -1,10 +1,29 @@
 const markdownInclude = require('markdown-include')
 const path = require('path')
+const glob = require('glob')
 const fs = require('fs')
 
 const rootPath = path.resolve(__dirname) + '/../../'
 const file = {}
 
+file.list = function (include = [], exclude = []) {
+  return new Promise(resolve => {
+    glob('channels/**/*.m3u', function (err, files) {
+      if (include.length) {
+        include = include.map(filename => `channels/${filename}.m3u`)
+        files = files.filter(filename => include.includes(filename))
+      }
+
+      if (exclude.length) {
+        exclude = exclude.map(filename => `channels/${filename}.m3u`)
+        files = files.filter(filename => !exclude.includes(filename))
+      }
+
+      resolve(files)
+    })
+  })
+}
+
 file.getFilename = function (filename) {
   return path.parse(filename).name
 }

scripts/helpers/parser.js

@@ -5,13 +5,6 @@ const file = require('./file')
 
 const parser = {}
 
-parser.parseIndex = function () {
-  const content = file.read('index.m3u')
-  const result = playlistParser.parse(content)
-
-  return result.items
-}
-
 parser.parsePlaylist = async function (url) {
   const content = file.read(url)
   const result = playlistParser.parse(content)

scripts/helpers/utils.js

@@ -74,22 +74,6 @@ utils.removeProtocol = function (string) {
   return string.replace(/(^\w+:|^)\/\//, '')
 }
 
-utils.filterPlaylists = function (arr, include = '', exclude = '') {
-  if (include) {
-    const included = include.split(',').map(filename => `channels/${filename}.m3u`)
-
-    return arr.filter(i => included.indexOf(i.url) > -1)
-  }
-
-  if (exclude) {
-    const excluded = exclude.split(',').map(filename => `channels/${filename}.m3u`)
-
-    return arr.filter(i => excluded.indexOf(i.url) === -1)
-  }
-
-  return arr
-}
-
 utils.sleep = function (ms) {
   return function (x) {
     return new Promise(resolve => setTimeout(() => resolve(x), ms))

scripts/remove-broken-links.js

@@ -1,15 +1,17 @@
 const parser = require('./helpers/parser')
+const file = require('./helpers/file')
 const log = require('./helpers/log')
 
 async function main() {
   log.start()
 
-  log.print(`Parsing 'index.m3u'...`)
-  const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
-  for (const playlist of playlists) {
-    log.print(`\nProcessing '${playlist.url}'...`)
+  let files = await file.list()
+  if (!files.length) log.print(`No files is selected\n`)
+  files = files.filter(file => file !== 'channels/unsorted.m3u')
+  for (const file of files) {
+    log.print(`\nProcessing '${file}'...`)
     await parser
-      .parsePlaylist(playlist.url)
+      .parsePlaylist(file)
       .then(removeBrokenLinks)
       .then(p => p.save())
   }

scripts/remove-duplicates.js

@@ -1,5 +1,6 @@
 const parser = require('./helpers/parser')
 const utils = require('./helpers/utils')
+const file = require('./helpers/file')
 const log = require('./helpers/log')
 
 let globalBuffer = []
@@ -7,18 +8,19 @@ let globalBuffer = []
 async function main() {
   log.start()
 
-  log.print(`Parsing 'index.m3u'...`)
-  const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
-  for (const playlist of playlists) {
-    log.print(`\nProcessing '${playlist.url}'...`)
+  let files = await file.list()
+  if (!files.length) log.print(`No files is selected\n`)
+  files = files.filter(file => file !== 'channels/unsorted.m3u')
+  for (const file of files) {
+    log.print(`\nProcessing '${file}'...`)
     await parser
-      .parsePlaylist(playlist.url)
+      .parsePlaylist(file)
       .then(addToGlobalBuffer)
       .then(removeDuplicates)
       .then(p => p.save())
   }
 
-  if (playlists.length) {
+  if (files.length) {
     log.print(`\nProcessing 'channels/unsorted.m3u'...`)
     await parser
       .parsePlaylist('channels/unsorted.m3u')

scripts/sort.js

@@ -1,16 +1,18 @@
 const parser = require('./helpers/parser')
 const utils = require('./helpers/utils')
+const file = require('./helpers/file')
 const log = require('./helpers/log')
 
 async function main() {
   log.start()
 
-  log.print(`Parsing 'index.m3u'...`)
-  let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
-  for (const playlist of playlists) {
-    log.print(`\nProcessing '${playlist.url}'...`)
+  let files = await file.list()
+  if (!files.length) log.print(`No files is selected\n`)
+  files = files.filter(file => file !== 'channels/unsorted.m3u')
+  for (const file of files) {
+    log.print(`\nProcessing '${file}'...`)
     await parser
-      .parsePlaylist(playlist.url)
+      .parsePlaylist(file)
       .then(sortChannels)
       .then(p => p.save())
   }