From fa47e80a8fd570d08ab55db6d9914439ef56bce0 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 05:16:45 +0300 Subject: [PATCH 01/14] Create /tmp directory --- scripts/.gitignore | 2 -- scripts/core/api.js | 2 +- scripts/core/db.js | 2 +- scripts/core/generator.js | 2 +- scripts/{data => tmp}/.gitignore | 2 +- 5 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 scripts/.gitignore rename scripts/{data => tmp}/.gitignore (92%) diff --git a/scripts/.gitignore b/scripts/.gitignore deleted file mode 100644 index 9becffee7..000000000 --- a/scripts/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/logs/ -/database/ \ No newline at end of file diff --git a/scripts/core/api.js b/scripts/core/api.js index 901f0ef26..abf6cf04b 100644 --- a/scripts/core/api.js +++ b/scripts/core/api.js @@ -1,7 +1,7 @@ const _ = require('lodash') const file = require('./file') -const DATA_DIR = process.env.DATA_DIR || './scripts/data' +const DATA_DIR = process.env.DATA_DIR || './scripts/tmp/data' class API { constructor(filepath) { diff --git a/scripts/core/db.js b/scripts/core/db.js index db3fd527f..3cde1161d 100644 --- a/scripts/core/db.js +++ b/scripts/core/db.js @@ -1,7 +1,7 @@ const nedb = require('nedb-promises') const file = require('./file') -const DB_DIR = process.env.DB_DIR || './scripts/database' +const DB_DIR = process.env.DB_DIR || './scripts/tmp/database' class Database { constructor(filepath) { diff --git a/scripts/core/generator.js b/scripts/core/generator.js index 77a769162..5f22e901b 100644 --- a/scripts/core/generator.js +++ b/scripts/core/generator.js @@ -4,7 +4,7 @@ const logger = require('./logger') const file = require('./file') const PUBLIC_DIR = process.env.PUBLIC_DIR || '.gh-pages' -const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs/generators' +const LOGS_DIR = process.env.LOGS_DIR || 'scripts/tmp/logs/generators' const generator = {} diff --git a/scripts/data/.gitignore b/scripts/tmp/.gitignore similarity index 92% rename from scripts/data/.gitignore rename to scripts/tmp/.gitignore index a3a0c8b5f..c96a04f00 100644 --- a/scripts/data/.gitignore +++ b/scripts/tmp/.gitignore @@ -1,2 +1,2 @@ -* +* !.gitignore \ No newline at end of file From 7850a93cf1bd42a2d5f5aca9ea5c9fd4a7a54792 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 05:17:17 +0300 Subject: [PATCH 02/14] Update template.md --- .readme/template.md | 3 --- tests/__data__/expected/_readme.md | 7 ------- 2 files changed, 10 deletions(-) diff --git a/.readme/template.md b/.readme/template.md index 74498a427..b27b593d0 100644 --- a/.readme/template.md +++ b/.readme/template.md @@ -1,14 +1,11 @@ # IPTV -[![auto-update](https://github.com/iptv-org/iptv/actions/workflows/auto-update.yml/badge.svg)](https://github.com/iptv-org/iptv/actions/workflows/auto-update.yml) - Collection of publicly available IPTV (Internet Protocol television) channels from all over the world. ## Table of contents - 🚀 [How to use?](#how-to-use) - 📺 [Playlists](#playlists) -- 🗓 [EPG](#epg) - 🗄 [Database](#database) - 👨‍💻 [API](#api) - 📚 [Resources](#resources) diff --git a/tests/__data__/expected/_readme.md b/tests/__data__/expected/_readme.md index 1e4cc2a5d..cd7ec179f 100644 --- a/tests/__data__/expected/_readme.md +++ b/tests/__data__/expected/_readme.md @@ -1,14 +1,11 @@ # IPTV -[![auto-update](https://github.com/iptv-org/iptv/actions/workflows/auto-update.yml/badge.svg)](https://github.com/iptv-org/iptv/actions/workflows/auto-update.yml) - Collection of publicly available IPTV (Internet Protocol television) channels from all over the world. ## Table of contents - 🚀 [How to use?](#how-to-use) - 📺 [Playlists](#playlists) -- 🗓 [EPG](#epg) - 🗄 [Database](#database) - 👨‍💻 [API](#api) - 📚 [Resources](#resources) @@ -454,10 +451,6 @@ Same thing, but split up into separate files: -## EPG - -The playlists already contain links to all guides, so players with support the `x-tvg-url` tag should load it automatically. Otherwise, you can choose one of the guides featured in the [iptv-org/epg](https://github.com/iptv-org/epg) repository. - ## Database All channel data is taken from the [iptv-org/database](https://github.com/iptv-org/database) repository. If you find any errors please open a new [issue](https://github.com/iptv-org/database/issues) there. From 61e555906149c8dfc0de45dd4a7c4053b6cfffef Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 05:17:44 +0300 Subject: [PATCH 03/14] Update tests/__data__ --- tests/__data__/expected/streams/ad.m3u | 5 +++++ tests/__data__/expected/streams/af.m3u | 5 +++++ tests/__data__/expected/streams/aw.m3u | 6 ++++++ tests/__data__/expected/streams/es.m3u | 7 +++++++ tests/__data__/expected/streams/nl.m3u | 11 ++++++++++ tests/__data__/expected/streams/pe.m3u | 3 +++ tests/__data__/expected/streams/ru.m3u | 9 +++++++++ tests/__data__/expected/streams/uk.m3u | 5 +++++ .../input/database/playlist_format.streams.db | 20 +++++++++++++++++++ 9 files changed, 71 insertions(+) create mode 100644 tests/__data__/expected/streams/ad.m3u create mode 100644 tests/__data__/expected/streams/af.m3u create mode 100644 tests/__data__/expected/streams/aw.m3u create mode 100644 tests/__data__/expected/streams/es.m3u create mode 100644 tests/__data__/expected/streams/nl.m3u create mode 100644 tests/__data__/expected/streams/pe.m3u create mode 100644 tests/__data__/expected/streams/ru.m3u create mode 100644 tests/__data__/expected/streams/uk.m3u create mode 100644 tests/__data__/input/database/playlist_format.streams.db diff --git a/tests/__data__/expected/streams/ad.m3u b/tests/__data__/expected/streams/ad.m3u new file mode 100644 index 000000000..5007b9ec0 --- /dev/null +++ b/tests/__data__/expected/streams/ad.m3u @@ -0,0 +1,5 @@ +#EXTM3U +#EXTINF:-1 tvg-id="AndorraTV.ad" user-agent="Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148",ATV (720p) [Offline] +#EXTVLCOPT:http-referrer=http://imn.iq +#EXTVLCOPT:http-user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 +https://iptv-all.lanesh4d0w.repl.co/andorra/atv diff --git a/tests/__data__/expected/streams/af.m3u b/tests/__data__/expected/streams/af.m3u new file mode 100644 index 000000000..976a1d532 --- /dev/null +++ b/tests/__data__/expected/streams/af.m3u @@ -0,0 +1,5 @@ +#EXTM3U +#EXTINF:-1 tvg-id="KayhanTV.af",Kayhan TV +http://208.93.117.113/live/Stream1/playlist.m3u8 +#EXTINF:-1 tvg-id="Sharq.af",Sharq +http://51.210.199.50/hls/stream.m3u8 diff --git a/tests/__data__/expected/streams/aw.m3u b/tests/__data__/expected/streams/aw.m3u new file mode 100644 index 000000000..3c2c2eb89 --- /dev/null +++ b/tests/__data__/expected/streams/aw.m3u @@ -0,0 +1,6 @@ +#EXTM3U +#EXTINF:-1 tvg-id="Telearuba.aw",Telearuba (720p) +http://cdn.setar.aw:1935/Telearuba/smil:telearuba.smil/playlist.m3u8 +#EXTINF:-1 tvg-id="Telearuba.aw" user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36",Telearuba (480p) [Not 24/7] +#EXTVLCOPT:http-user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36 +https://backend-server-dot-telearuba-app.appspot.com/media/livestream13/playlist.m3u8 diff --git a/tests/__data__/expected/streams/es.m3u b/tests/__data__/expected/streams/es.m3u new file mode 100644 index 000000000..9b94dc893 --- /dev/null +++ b/tests/__data__/expected/streams/es.m3u @@ -0,0 +1,7 @@ +#EXTM3U +#EXTINF:-1 tvg-id="",Caillou +https://dhx-caillou-1-es.samsung.wurl.tv/playlist.m3u8 +#EXTINF:-1 tvg-id="",iHola Play +https://rakuten-hola-2-es.samsung.wurl.tv/playlist.m3u8 +#EXTINF:-1 tvg-id="",Planeta Junior TV +https://deaplaneta-planetakidz-1-es.samsung.wurl.tv/playlist.m3u8 diff --git a/tests/__data__/expected/streams/nl.m3u b/tests/__data__/expected/streams/nl.m3u new file mode 100644 index 000000000..1eb378725 --- /dev/null +++ b/tests/__data__/expected/streams/nl.m3u @@ -0,0 +1,11 @@ +#EXTM3U +#EXTINF:-1 tvg-id="NPO1.nl",NPO 1 (342p) [Geo-blocked] +http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo1/npo1.isml/.m3u8 +#EXTINF:-1 tvg-id="NPO1.nl",NPO 1 (1080p) [Geo-blocked] +http://stream.tvtap.net:8081/live/nl-npo1.stream/30fps.m3u8 +#EXTINF:-1 tvg-id="NPO1.nl",NPO 1 (1080p) [Geo-blocked] +http://stream.tvtap.net:8081/live/nl-npo1.stream/60fps.m3u8 +#EXTINF:-1 tvg-id="NPO2.nl",NPO 2 (342p) +http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo2/npo2.isml/.m3u8 +#EXTINF:-1 tvg-id="NPO2.nl",NPO 2 (302p) [Geo-blocked] +http://stream.tvtap.net:8081/live/nl-npo2.stream/playlist.m3u8 diff --git a/tests/__data__/expected/streams/pe.m3u b/tests/__data__/expected/streams/pe.m3u new file mode 100644 index 000000000..a2de651e5 --- /dev/null +++ b/tests/__data__/expected/streams/pe.m3u @@ -0,0 +1,3 @@ +#EXTM3U +#EXTINF:-1 tvg-id="Tele2000.pe",Tele 2000 +https://servilive.com:3126/live/tele2000live.m3u8 diff --git a/tests/__data__/expected/streams/ru.m3u b/tests/__data__/expected/streams/ru.m3u new file mode 100644 index 000000000..cf2288e05 --- /dev/null +++ b/tests/__data__/expected/streams/ru.m3u @@ -0,0 +1,9 @@ +#EXTM3U +#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p) +http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8 +#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p) +http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8 +#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p) +http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8 +#EXTINF:-1 tvg-id="LDPRTV.ru",ЛДПР ТВ (1080p) +http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8 diff --git a/tests/__data__/expected/streams/uk.m3u b/tests/__data__/expected/streams/uk.m3u new file mode 100644 index 000000000..c1deeea97 --- /dev/null +++ b/tests/__data__/expected/streams/uk.m3u @@ -0,0 +1,5 @@ +#EXTM3U +#EXTINF:-1 tvg-id="BBCNews.uk",BBC News HD (720p) [Not 24/7] +http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8 +#EXTINF:-1 tvg-id="BBCNews.uk",BBC News HD (480p) [Geo-blocked] +http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8 diff --git a/tests/__data__/input/database/playlist_format.streams.db b/tests/__data__/input/database/playlist_format.streams.db new file mode 100644 index 000000000..ba681f59f --- /dev/null +++ b/tests/__data__/input/database/playlist_format.streams.db @@ -0,0 +1,20 @@ +{"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF3"} +{"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF0"} +{"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF1"} +{"title":"ЛДПР ТВ (1080p)","channel":"LDPRTV.ru","filepath":"tests/__data__/output/streams/ru.m3u","url":"http://46.46.143.222:1935/live/mp4:ldpr.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2ST8btby3mmsgPF2"} +{"title":"BBC News HD (720p) [Not 24/7]","channel":"BBCNews.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/index.m3u8","http_referrer":null,"user_agent":null,"_id":"3TbieV1ptnZVCIdn"} +{"title":"ATV (720p) [Offline]","channel":"AndorraTV.ad","filepath":"tests/__data__/output/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","_id":"I6cjG2xCBRFFP4sz"} +{"title":"BBC News HD (480p) [Geo-blocked]","channel":"BBCNews.uk","filepath":"tests/__data__/output/streams/uk.m3u","url":"http://1111296894.rsc.cdn77.org/LS-ATL-54548-6/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"WTbieV1ptnZVCIdn"} +{"title":"Kayhan TV","channel":"KayhanTV.af","filepath":"tests/__data__/output/streams/af.m3u","url":"http://208.93.117.113/live/Stream1/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"cFFpFVzSn6xFMUF3"} +{"title":"Sharq","channel":"Sharq.af","filepath":"tests/__data__/output/streams/af.m3u","url":"http://51.210.199.50/hls/stream.m3u8","http_referrer":null,"user_agent":null,"_id":"u7iyA6cjtf1iWWAZ"} +{"channel":"NPO1.nl","title":"NPO 1 (342p) [Geo-blocked]","filepath":"tests/__data__/output/streams/nl.m3u","url":"http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo1/npo1.isml/.m3u8","http_referrer":null,"user_agent":null,"_id":"mvUyDVuS5gc8gLJV"} +{"channel":"NPO1.nl","title":"NPO 1 (1080p) [Geo-blocked]","filepath":"tests/__data__/output/streams/nl.m3u","url":"http://stream.tvtap.net:8081/live/nl-npo1.stream/30fps.m3u8","http_referrer":null,"user_agent":null,"_id":"8WVbsxsYeOL7kHQl"} +{"channel":"NPO1.nl","title":"NPO 1 (1080p) [Geo-blocked]","filepath":"tests/__data__/output/streams/nl.m3u","url":"http://stream.tvtap.net:8081/live/nl-npo1.stream/60fps.m3u8","http_referrer":null,"user_agent":null,"_id":"8WVbsxsYeOL7kHQB"} +{"channel":"NPO2.nl","title":"NPO 2 (342p)","filepath":"tests/__data__/output/streams/nl.m3u","url":"http://resolver.streaming.api.nos.nl/livestream?url=/live/npo/tvlive/npo2/npo2.isml/.m3u8","http_referrer":null,"user_agent":null,"_id":"2p1TNGO0mF0MJOGy"} +{"channel":"NPO2.nl","title":"NPO 2 (302p) [Geo-blocked]","filepath":"tests/__data__/output/streams/nl.m3u","url":"http://stream.tvtap.net:8081/live/nl-npo2.stream/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"nhL85BL7YM5OR7cn"} +{"title":"Tele 2000","channel":"Tele2000.pe","filepath":"tests/__data__/output/streams/pe.m3u","url":"https://servilive.com:3126/live/tele2000live.m3u8","http_referrer":null,"user_agent":null,"_id":"cF0pFVzSn6xFMUF3"} +{"title":"Planeta Junior TV","channel":"","filepath":"tests/__data__/output/streams/es.m3u","url":"https://deaplaneta-planetakidz-1-es.samsung.wurl.tv/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"1BT8btby3mmsgPF0"} +{"title":"Caillou","channel":"","filepath":"tests/__data__/output/streams/es.m3u","url":"https://dhx-caillou-1-es.samsung.wurl.tv/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"3BT8btby3mmsgPF0"} +{"title":"iHola Play","channel":"","filepath":"tests/__data__/output/streams/es.m3u","url":"https://rakuten-hola-2-es.samsung.wurl.tv/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"2BT8btby3mmsgPF0"} +{"title":"Telearuba (720p)","channel":"Telearuba.aw","filepath":"tests/__data__/output/streams/aw.m3u","url":"http://cdn.setar.aw:1935/Telearuba/smil:telearuba.smil/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"6BT8btby3mmsgPF0"} +{"title":"Telearuba (480p) [Not 24/7]","channel":"Telearuba.aw","filepath":"tests/__data__/output/streams/aw.m3u","url":"https://backend-server-dot-telearuba-app.appspot.com/media/livestream13/playlist.m3u8","http_referrer":null,"user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36","_id":"4BT8btby3mmsgPF0"} \ No newline at end of file From 78038b950513bece2c5e460e1633349f4fc683ec Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 05:17:50 +0300 Subject: [PATCH 04/14] Create format.test.js --- tests/commands/playlist/format.test.js | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/commands/playlist/format.test.js diff --git a/tests/commands/playlist/format.test.js b/tests/commands/playlist/format.test.js new file mode 100644 index 000000000..0d048732a --- /dev/null +++ b/tests/commands/playlist/format.test.js @@ -0,0 +1,32 @@ +const { execSync } = require('child_process') +const fs = require('fs-extra') +const path = require('path') +const glob = require('glob') + +beforeEach(() => { + fs.emptyDirSync('tests/__data__/output') + fs.copyFileSync( + 'tests/__data__/input/database/playlist_format.streams.db', + 'tests/__data__/output/streams.db' + ) + + const stdout = execSync('DB_DIR=tests/__data__/output npm run playlist:format', { + encoding: 'utf8' + }) +}) + +it('can format playlists', () => { + const files = glob + .sync('tests/__data__/expected/streams/*.m3u') + .map(f => f.replace('tests/__data__/expected/', '')) + + files.forEach(filepath => { + expect(content(`output/${filepath}`), filepath).toBe(content(`expected/${filepath}`)) + }) +}) + +function content(filepath) { + return fs.readFileSync(`tests/__data__/${filepath}`, { + encoding: 'utf8' + }) +} From 4a8afcc0ebc7184aacb05d62a9105233e6fd5579 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 05:17:54 +0300 Subject: [PATCH 05/14] Create format.js --- scripts/commands/playlist/format.js | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 scripts/commands/playlist/format.js diff --git a/scripts/commands/playlist/format.js b/scripts/commands/playlist/format.js new file mode 100644 index 000000000..08f3ec921 --- /dev/null +++ b/scripts/commands/playlist/format.js @@ -0,0 +1,26 @@ +const { create: createPlaylist } = require('../../core/playlist') +const { db, logger, file } = require('../../core') +const { orderBy } = require('natural-orderby') +const _ = require('lodash') + +async function main() { + logger.info('loading streams...') + await db.streams.load() + let streams = await db.streams.find({}) + + logger.info('sorting links...') + streams = orderBy( + streams, + ['channel', s => (s.channel ? '' : s.title), 'url'], + ['asc', 'asc', 'asc'] + ) + + logger.info('saving...') + const files = _.groupBy(streams, 'filepath') + for (const filepath in files) { + const playlist = createPlaylist(files[filepath], { public: false }) + await file.create(filepath, playlist.toString()) + } +} + +main() From e4f1d178e3fafe620b75aa654038b54e9672957d Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 05:22:01 +0300 Subject: [PATCH 06/14] Update package.json --- scripts/commands/api/load.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/commands/api/load.sh b/scripts/commands/api/load.sh index b26a0c3c2..37fd89619 100755 --- a/scripts/commands/api/load.sh +++ b/scripts/commands/api/load.sh @@ -1,10 +1,10 @@ #!/bin/bash -mkdir -p scripts/data -curl -L -o scripts/data/blocklist.json https://iptv-org.github.io/api/blocklist.json -curl -L -o scripts/data/categories.json https://iptv-org.github.io/api/categories.json -curl -L -o scripts/data/channels.json https://iptv-org.github.io/api/channels.json -curl -L -o scripts/data/countries.json https://iptv-org.github.io/api/countries.json -curl -L -o scripts/data/languages.json https://iptv-org.github.io/api/languages.json -curl -L -o scripts/data/regions.json https://iptv-org.github.io/api/regions.json -curl -L -o scripts/data/subdivisions.json https://iptv-org.github.io/api/subdivisions.json \ No newline at end of file +mkdir -p scripts/tmp/data +curl -L -o scripts/tmp/data/blocklist.json https://iptv-org.github.io/api/blocklist.json +curl -L -o scripts/tmp/data/categories.json https://iptv-org.github.io/api/categories.json +curl -L -o scripts/tmp/data/channels.json https://iptv-org.github.io/api/channels.json +curl -L -o scripts/tmp/data/countries.json https://iptv-org.github.io/api/countries.json +curl -L -o scripts/tmp/data/languages.json https://iptv-org.github.io/api/languages.json +curl -L -o scripts/tmp/data/regions.json https://iptv-org.github.io/api/regions.json +curl -L -o scripts/tmp/data/subdivisions.json https://iptv-org.github.io/api/subdivisions.json \ No newline at end of file From 62ec9c9a62635acdb32f49a44523607260ff2c1c Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 05:22:17 +0300 Subject: [PATCH 07/14] Update package.json --- package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index b816c95d4..e8823fe48 100644 --- a/package.json +++ b/package.json @@ -5,14 +5,16 @@ "api:generate": "node scripts/commands/api/generate.js", "api:deploy": "npx gh-pages -a -m \"Deploy to iptv-org/api\" -d .api -r https://$GITHUB_TOKEN@github.com/iptv-org/api.git -x", "db:create": "node scripts/commands/database/create.js", + "playlist:format": "node scripts/commands/playlist/format.js", "playlist:generate": "node scripts/commands/playlist/generate.js", "playlist:validate": "node scripts/commands/playlist/validate.js", "playlist:lint": "npx m3u-linter -c m3u-linter.json", "playlist:deploy": "npx gh-pages -a -m \"Deploy to GitHub Pages\" -d .gh-pages -r https://$GITHUB_TOKEN@github.com/iptv-org/iptv.git -x", "readme:update": "node scripts/commands/readme/update.js", + "format": "npm run api:load && npm run db:create && npm run playlist:format", + "check": "npm run api:load && npm run playlist:lint && npm run playlist:validate", "update": "npm run api:load && npm run db:create && npm run playlist:generate && npm run api:generate && npm run readme:update", "deploy": "npm run playlist:deploy && npm run api:deploy", - "check": "npm run api:load && npm run playlist:lint && npm run playlist:validate", "test": "jest --runInBand" }, "jest": { From 06f29decc1257dd50bfd7aac9e5f43403979746e Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 05:24:52 +0300 Subject: [PATCH 08/14] Update update.js --- scripts/commands/readme/update.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/commands/readme/update.js b/scripts/commands/readme/update.js index 3d2b969a0..0d0c05b56 100644 --- a/scripts/commands/readme/update.js +++ b/scripts/commands/readme/update.js @@ -2,7 +2,7 @@ const { file, markdown, parser, logger, api } = require('../../core') const { create: createTable } = require('../../core/table') const { program } = require('commander') -const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs/generators' +const LOGS_DIR = process.env.LOGS_DIR || 'scripts/tmp/logs/generators' const options = program .option('-c, --config ', 'Set path to config file', '.readme/config.json') From ccf635673ad7e132573ef63300b4270d8ec1782a Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 05:48:05 +0300 Subject: [PATCH 09/14] Update tests/__data__ --- .../__data__/expected/database/db_create.streams.db | 13 +++++++------ tests/__data__/expected/streams/af.m3u | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/__data__/expected/database/db_create.streams.db b/tests/__data__/expected/database/db_create.streams.db index 33915d8a0..5ea1bca76 100644 --- a/tests/__data__/expected/database/db_create.streams.db +++ b/tests/__data__/expected/database/db_create.streams.db @@ -1,6 +1,7 @@ -{"channel":null,"title":"TVN","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"_id":"UAYbCvjOWvqXHH95"} -{"channel":"LibyasChannel.ly","title":"Libyas Channel","filepath":"tests/__data__/input/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"_id":"I0rJlwp3rZEy2SnG"} -{"channel":"EverydayHeroes.us","title":"Everyday Heroes (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","http_referrer":null,"user_agent":null,"_id":"tdzk1IN7wLJxfGab"} -{"channel":null,"title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","_id":"3evMHt3nFsZjhzas"} -{"channel":null,"title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"_id":"agJlTEr8wwpbWgw0"} -{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"FpMhoWjtpnVuYlO9"} +{"channel":"ATV.ad","title":"ATV (720p) [Offline]","filepath":"tests/__data__/input/streams/ad.m3u","url":"https://iptv-all.lanesh4d0w.repl.co/andorra/atv","http_referrer":"http://imn.iq","user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148","_id":"k4XpZHQAyqyTbcf0"} +{"channel":"LibyasChannel.ly","title":"Libyas Channel","filepath":"tests/__data__/input/streams/ly.m3u","url":"https://master.starmena-cloud.com/hls/libyas.m3u8","http_referrer":null,"user_agent":null,"_id":"ki4YjAoNNoIY8sSm"} +{"channel":null,"title":"1A Network (720p)","filepath":"tests/__data__/input/streams/unsorted.m3u","url":"https://simultv.s.llnwi.net/n4s4/2ANetwork/interlink.m3u8","http_referrer":null,"user_agent":null,"_id":"IZpCJjjWPaBYh7Dr"} +{"channel":null,"title":"Fox Sports 2 Asia (Thai) (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist.m3u8","http_referrer":null,"user_agent":null,"_id":"6c4J4vs8K69wMJ7S"} +{"channel":null,"title":"TVN","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"_id":"xEmbX384v3t3F5Wg"} +{"channel":"EverydayHeroes.us","title":"Everyday Heroes (720p)","filepath":"tests/__data__/input/streams/us_blocked.m3u","url":"https://a.jsrdn.com/broadcast/7b1451fa52/+0000/c.m3u8","http_referrer":null,"user_agent":null,"_id":"BZsRPt8VS4kIJnfi"} +{"channel":"qib22lAq1L.us","title":"ABC (720p)","filepath":"tests/__data__/input/streams/wrong_id.m3u","url":"https://example.com/playlist2.m3u8","http_referrer":null,"user_agent":null,"_id":"eFUlUnST5zJSBWAF"} diff --git a/tests/__data__/expected/streams/af.m3u b/tests/__data__/expected/streams/af.m3u index 976a1d532..cff8ac5e0 100644 --- a/tests/__data__/expected/streams/af.m3u +++ b/tests/__data__/expected/streams/af.m3u @@ -1,5 +1,5 @@ #EXTM3U #EXTINF:-1 tvg-id="KayhanTV.af",Kayhan TV http://208.93.117.113/live/Stream1/playlist.m3u8 -#EXTINF:-1 tvg-id="Sharq.af",Sharq +#EXTINF:-1 tvg-id="",Sharq http://51.210.199.50/hls/stream.m3u8 From bee42ff2d597a8b51eb53820f6c144aafcb1906f Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 05:48:11 +0300 Subject: [PATCH 10/14] Update create.js --- scripts/commands/database/create.js | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/scripts/commands/database/create.js b/scripts/commands/database/create.js index 4b450c084..5cce0dd19 100644 --- a/scripts/commands/database/create.js +++ b/scripts/commands/database/create.js @@ -1,4 +1,4 @@ -const { db, file, parser, store, logger, api } = require('../../core') +const { db, file, parser, store, logger } = require('../../core') const { program } = require('commander') const _ = require('lodash') @@ -8,21 +8,7 @@ const options = program .opts() async function main() { - logger.info('starting...') - - await saveToDatabase(await findStreams()) -} - -main() - -async function findStreams() { - logger.info(`loading channels...`) - await api.channels.load() - const channels = _.keyBy(await api.channels.all(), 'id') - logger.info(`looking for streams...`) - await db.streams.load() - const streams = [] const files = await file.list(`${options.inputDir}/**/*.m3u`) for (const filepath of files) { @@ -31,9 +17,8 @@ async function findStreams() { item.filepath = filepath const stream = store.create() - const channel = channels[item.tvg.id] - stream.set('channel', { channel: channel ? channel.id : null }) + stream.set('channel', { channel: item.tvg.id }) stream.set('title', { title: item.name }) stream.set('filepath', { filepath: item.filepath }) stream.set('url', { url: item.url }) @@ -45,13 +30,11 @@ async function findStreams() { } logger.info(`found ${streams.length} streams`) - return streams -} - -async function saveToDatabase(streams = []) { logger.info('saving to the database...') - + await db.streams.load() await db.streams.reset() const data = streams.map(stream => stream.data()) await db.streams.insert(data) } + +main() From 7285887f01b356edc8689b9f22f186bc639cc3bc Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 05:48:21 +0300 Subject: [PATCH 11/14] Update format.js --- scripts/commands/playlist/format.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/commands/playlist/format.js b/scripts/commands/playlist/format.js index 08f3ec921..557a77a66 100644 --- a/scripts/commands/playlist/format.js +++ b/scripts/commands/playlist/format.js @@ -1,5 +1,5 @@ const { create: createPlaylist } = require('../../core/playlist') -const { db, logger, file } = require('../../core') +const { db, api, logger, file } = require('../../core') const { orderBy } = require('natural-orderby') const _ = require('lodash') @@ -8,6 +8,21 @@ async function main() { await db.streams.load() let streams = await db.streams.find({}) + logger.info(`loading channels...`) + await api.channels.load() + const channels = _.keyBy(await api.channels.all(), 'id') + + logger.info('removing invalid tvg-id...') + streams = streams.map(stream => { + const channel = channels[stream.channel] + + if (!channel) { + stream.channel = null + } + + return stream + }) + logger.info('sorting links...') streams = orderBy( streams, From 77f4fac6c37a44db7e762c58956c9647387e26be Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 05:59:46 +0300 Subject: [PATCH 12/14] Update af.m3u --- tests/__data__/expected/streams/af.m3u | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__data__/expected/streams/af.m3u b/tests/__data__/expected/streams/af.m3u index cff8ac5e0..976a1d532 100644 --- a/tests/__data__/expected/streams/af.m3u +++ b/tests/__data__/expected/streams/af.m3u @@ -1,5 +1,5 @@ #EXTM3U #EXTINF:-1 tvg-id="KayhanTV.af",Kayhan TV http://208.93.117.113/live/Stream1/playlist.m3u8 -#EXTINF:-1 tvg-id="",Sharq +#EXTINF:-1 tvg-id="Sharq.af",Sharq http://51.210.199.50/hls/stream.m3u8 From 8bf2e8f157be9573aff3edbf5577148003c5279c Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 06:00:10 +0300 Subject: [PATCH 13/14] Update format.js --- scripts/commands/playlist/format.js | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/scripts/commands/playlist/format.js b/scripts/commands/playlist/format.js index 557a77a66..08f3ec921 100644 --- a/scripts/commands/playlist/format.js +++ b/scripts/commands/playlist/format.js @@ -1,5 +1,5 @@ const { create: createPlaylist } = require('../../core/playlist') -const { db, api, logger, file } = require('../../core') +const { db, logger, file } = require('../../core') const { orderBy } = require('natural-orderby') const _ = require('lodash') @@ -8,21 +8,6 @@ async function main() { await db.streams.load() let streams = await db.streams.find({}) - logger.info(`loading channels...`) - await api.channels.load() - const channels = _.keyBy(await api.channels.all(), 'id') - - logger.info('removing invalid tvg-id...') - streams = streams.map(stream => { - const channel = channels[stream.channel] - - if (!channel) { - stream.channel = null - } - - return stream - }) - logger.info('sorting links...') streams = orderBy( streams, From adbf764c3e3ed4f2039d335b7b68011bac4c24a8 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 30 Apr 2023 06:00:12 +0300 Subject: [PATCH 14/14] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e8823fe48..51b7c492b 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "playlist:lint": "npx m3u-linter -c m3u-linter.json", "playlist:deploy": "npx gh-pages -a -m \"Deploy to GitHub Pages\" -d .gh-pages -r https://$GITHUB_TOKEN@github.com/iptv-org/iptv.git -x", "readme:update": "node scripts/commands/readme/update.js", - "format": "npm run api:load && npm run db:create && npm run playlist:format", + "format": "npm run db:create && npm run playlist:format", "check": "npm run api:load && npm run playlist:lint && npm run playlist:validate", "update": "npm run api:load && npm run db:create && npm run playlist:generate && npm run api:generate && npm run readme:update", "deploy": "npm run playlist:deploy && npm run api:deploy",