Merge pull request #3532 from iptv-org/create-filter-js

Update blacklist.json

.github/workflows/auto-update.yml

@@ -18,9 +18,26 @@ jobs:
         with:
           name: channels
           path: channels/
-  format:
+  filter:
     runs-on: ubuntu-latest
     needs: remove-duplicates
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Download Artifacts
+        uses: actions/download-artifact@v2
+      - name: Install Dependencies
+        run: npm install
+      - name: Filter Playlists
+        run: node scripts/filter.js
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: channels
+          path: channels/
+  format:
+    runs-on: ubuntu-latest
+    needs: filter
     steps:
       - name: Checkout
         uses: actions/checkout@v2

CONTRIBUTING.md

@@ -171,7 +171,8 @@ http://example.com/stream.m3u8
   - `categories.json`: list of supported categories.
   - `clean.js`: used in GitHub Action to check all links and remove broken ones.
   - `db.js`: contains functions for retrieving and managing the channel list.
-  - `format.js`: used within GitHub Action to sort channels and remove duplicates from each playlist.
+  - `filter.js`: used within GitHub Action to remove blacklisted channels from playlists.
+  - `format.js`: used within GitHub Action to format channel descriptions and sort playlists.
   - `generate.js`: used within GitHub Action to generate all additional playlists.
   - `parser.js`: contains functions for parsing playlists.
   - `regions.json`: list of supported region codes.

scripts/blacklist.json

@@ -1,69 +1,380 @@
 [
-  "Animal Planet",
+  {
-  "Asian Food Network",
+    "name": "Animal Planet",
-  "Cooking Channel",
+    "country": "us",
-  "Discovery Asia",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
-  "Discovery Channel",
+    "aliases": []
-  "Discovery Civiliztion",
+  },
-  "Discovery en Espanol",
+  {
-  "Discovery Family",
+    "name": "Asian Food Network",
-  "Discovery Historia",
+    "country": "sg",
-  "Discovery History",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
-  "Discovery Home and Health",
+    "aliases": []
-  "Discovery Life",
+  },
-  "Discovery Science",
+  {
-  "Discovery Shed",
+    "name": "Cooking Channel",
-  "Discovery Theater",
+    "country": "us",
-  "Discovery Travel and Living",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
-  "Discovery Turbo Xtra",
+    "aliases": []
-  "Discovery World",
+  },
-  "Discovery",
+  {
-  "DIY Network",
+    "name": "Discovery Asia",
-  "DKiss",
+    "country": "us",
-  "DMax",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
-  "Eurosport 1",
+    "aliases": []
-  "Eurosport 2",
+  },
-  "Eurosport",
+  {
-  "eve",
+    "name": "Discovery Channel",
-  "Familia Discovery",
+    "country": "us",
-  "Fatafeat",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
-  "FEM",
+    "aliases": []
-  "Fine Living",
+  },
-  "Food Network",
+  {
-  "food tv",
+    "name": "Discovery Civiliztion",
-  "foodtv",
+    "country": "us",
-  "Frisbee",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
-  "Giallo",
+    "aliases": []
-  "Golf TV",
+  },
-  "GolfTV",
+  {
-  "HGTV",
+    "name": "Discovery en Espanol",
-  "ID Investigation Discovery",
+    "country": "us",
-  "ID Investigation",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
-  "K2",
+    "aliases": []
-  "Living Channel",
+  },
-  "Mango",
+  {
-  "Motortrend",
+    "name": "Discovery Family",
-  "Nove",
+    "country": "us",
-  "Oprah",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
-  "OWN",
+    "aliases": []
-  "Quest Red",
+  },
-  "Quest",
+  {
-  "Real Time",
+    "name": "Discovery Historia",
-  "Sky TG24",
+    "country": "us",
-  "TLC",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
-  "Trvl Channel",
+    "aliases": []
-  "TTV",
+  },
-  "TV Norge",
+  {
-  "Tvn 24 Bis",
+    "name": "Discovery History",
-  "TVN 24",
+    "country": "us",
-  "Tvn 7",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
-  "TVN Extra",
+    "aliases": []
-  "TVN Fabula",
+  },
-  "TVN Meteo",
+  {
-  "TVN Style",
+    "name": "Discovery Home and Health",
-  "TVN Turbo",
+    "country": "us",
-  "TVN Warszawa",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
-  "TVN",
+    "aliases": []
-  "Vox",
+  },
-  "World Discovery"
+  {
+    "name": "Discovery Life",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Discovery Science",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Discovery Shed",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Discovery Theater",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Discovery Travel and Living",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Discovery Turbo Xtra",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Discovery World",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Discovery",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "DIY Network",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "DKiss",
+    "country": "es",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "DMax",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Eurosport 1",
+    "country": "fr",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Eurosport 2",
+    "country": "fr",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Eurosport",
+    "country": "fr",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "eve",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Familia Discovery",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Fatafeat",
+    "country": "eg",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "FEM",
+    "country": "no",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Fine Living",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Food Network",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "food tv",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": ["foodtv"]
+  },
+  {
+    "name": "Frisbee",
+    "country": "it",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Giallo",
+    "country": "it",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "GolfTV",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": ["Golf TV"]
+  },
+  {
+    "name": "HGTV",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Investigation Discovery",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": ["ID Investigation Discovery", "ID Investigation", "ID"]
+  },
+  {
+    "name": "K2",
+    "country": "it",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Living Channel",
+    "country": "nz",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Mango",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Motortrend",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Nove",
+    "country": "it",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "OWN",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": ["Oprah"]
+  },
+  {
+    "name": "Quest Red",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Quest",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Real Time",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Sky TG24",
+    "country": "it",
+    "dmca_notice": "https://github.com/iptv-org/iptv/pull/2294",
+    "aliases": []
+  },
+  {
+    "name": "TLC",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Trvl Channel",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "TTV",
+    "country": "pl",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "TV Norge",
+    "country": "no",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Tvn 24 Bis",
+    "country": "pl",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "TVN 24",
+    "country": "pl",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Tvn 7",
+    "country": "pl",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "TVN Extra",
+    "country": "pl",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "TVN Fabula",
+    "country": "pl",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "TVN Meteo",
+    "country": "pl",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "TVN Style",
+    "country": "pl",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "TVN Turbo",
+    "country": "pl",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "TVN Warszawa",
+    "country": "pl",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "TVN",
+    "country": "pl",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "Vox",
+    "country": "no",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  },
+  {
+    "name": "World Discovery",
+    "country": "us",
+    "dmca_notice": "https://github.com/iptv-org/iptv/issues/1831",
+    "aliases": []
+  }
 ]

scripts/filter.js

@@ -0,0 +1,67 @@
+const parser = require('./parser')
+const utils = require('./utils')
+const blacklist = require('./blacklist.json')
+
+async function main() {
+  const playlists = parseIndex()
+  for (const playlist of playlists) {
+    await loadPlaylist(playlist.url).then(removeBlacklisted).then(savePlaylist).then(done)
+  }
+
+  finish()
+}
+
+function parseIndex() {
+  console.info(`Parsing 'index.m3u'...`)
+  let playlists = parser.parseIndex()
+  console.info(`Found ${playlists.length} playlist(s)\n`)
+
+  return playlists
+}
+
+async function loadPlaylist(url) {
+  console.info(`Processing '${url}'...`)
+  return parser.parsePlaylist(url)
+}
+
+async function removeBlacklisted(playlist) {
+  console.info(`  Looking for blacklisted channels...`)
+  playlist.channels = playlist.channels.filter(channel => {
+    return !blacklist.find(i => {
+      const channelName = channel.name.toLowerCase()
+      return (
+        (i.name.toLowerCase() === channelName ||
+          i.aliases.map(i => i.toLowerCase()).includes(channelName)) &&
+        i.country === channel.filename
+      )
+    })
+  })
+
+  return playlist
+}
+
+async function savePlaylist(playlist) {
+  console.info(`  Saving playlist...`)
+  const original = utils.readFile(playlist.url)
+  const output = playlist.toString({ raw: true })
+
+  if (original === output) {
+    console.info(`No changes have been made.`)
+    return false
+  } else {
+    utils.createFile(playlist.url, output)
+    console.info(`Playlist has been updated.`)
+  }
+
+  return true
+}
+
+async function done() {
+  console.info(` `)
+}
+
+function finish() {
+  console.info('Done.')
+}
+
+main()

scripts/format.js

@@ -1,5 +1,4 @@
 const { program } = require('commander')
-const blacklist = require('./blacklist')
 const parser = require('./parser')
 const utils = require('./utils')
 const axios = require('axios')
@@ -35,7 +34,6 @@ async function main() {
   for (const playlist of playlists) {
     await loadPlaylist(playlist.url)
       .then(sortChannels)
-      .then(filterChannels)
       .then(detectResolution)
       .then(savePlaylist)
       .then(done)
@@ -67,16 +65,6 @@ async function sortChannels(playlist) {
   return playlist
 }
 
-async function filterChannels(playlist) {
-  console.info(`  Filtering channels...`)
-  const list = blacklist.map(i => i.toLowerCase())
-  playlist.channels = playlist.channels.filter(i => {
-    return !list.includes(i.name.toLowerCase())
-  })
-
-  return playlist
-}
-
 async function detectResolution(playlist) {
   if (!config.resolution) return playlist
   console.log('  Detecting resolution...')