Merge pull request #4034 from iptv-org/update-format-script

Update "format" script
This commit is contained in:
Dum4G 2021-08-10 20:08:30 +03:00 committed by GitHub
commit c6be0e810c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 227 additions and 579 deletions

View File

@ -20,90 +20,6 @@ jobs:
format: format:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: create-branch needs: create-branch
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: bot/auto-update
- name: Install Dependencies
run: npm install
- name: Format Playlists
run: node scripts/format.js
- name: Commit Changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: '[Bot] Formate playlists'
commit_user_name: iptv-bot
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
branch: bot/auto-update
file_pattern: channels/*
sort:
runs-on: ubuntu-latest
needs: format
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: bot/auto-update
- name: Install Dependencies
run: npm install
- name: Sort Channels
run: node scripts/sort.js
- name: Commit Changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: '[Bot] Sort channels'
commit_user_name: iptv-bot
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
branch: bot/auto-update
file_pattern: channels/*
remove-duplicates:
runs-on: ubuntu-latest
needs: sort
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: bot/auto-update
- name: Install Dependencies
run: npm install
- name: Remove Duplicates
run: node scripts/remove-duplicates.js
- name: Commit Changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: '[Bot] Remove duplicates'
commit_user_name: iptv-bot
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
branch: bot/auto-update
file_pattern: channels/*
filter:
runs-on: ubuntu-latest
needs: remove-duplicates
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: bot/auto-update
- name: Install Dependencies
run: npm install
- name: Filter Playlists
run: node scripts/filter.js
- name: Commit Changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: '[Bot] Filter channels'
commit_user_name: iptv-bot
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
branch: bot/auto-update
file_pattern: channels/*
detect-resolution:
runs-on: ubuntu-latest
needs: filter
continue-on-error: true continue-on-error: true
strategy: strategy:
fail-fast: false fail-fast: false
@ -277,10 +193,12 @@ jobs:
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
ref: bot/auto-update ref: bot/auto-update
- name: Setup FFmpeg
uses: FedericoCarboni/setup-ffmpeg@v1
- name: Install Dependencies - name: Install Dependencies
run: npm install run: npm install
- name: Detect Resolution - name: Format Playlists
run: node scripts/detect-resolution.js --country=${{ matrix.country }} run: node scripts/format.js --country=${{ matrix.country }} --status --resolution --debug
- name: Upload Artifact - name: Upload Artifact
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v2
with: with:
@ -288,7 +206,7 @@ jobs:
path: channels/${{ matrix.country }}.m3u path: channels/${{ matrix.country }}.m3u
commit-changes: commit-changes:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: detect-resolution needs: format
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v2
@ -302,7 +220,70 @@ jobs:
- name: Commit Changes - name: Commit Changes
uses: stefanzweifel/git-auto-commit-action@v4 uses: stefanzweifel/git-auto-commit-action@v4
with: with:
commit_message: '[Bot] Detect resolution' commit_message: '[Bot] Format playlists'
commit_user_name: iptv-bot
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
branch: bot/auto-update
file_pattern: channels/*
remove-duplicates:
runs-on: ubuntu-latest
needs: commit-changes
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: bot/auto-update
- name: Install Dependencies
run: npm install
- name: Remove Duplicates
run: node scripts/remove-duplicates.js
- name: Commit Changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: '[Bot] Remove duplicates'
commit_user_name: iptv-bot
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
branch: bot/auto-update
file_pattern: channels/*
sort:
runs-on: ubuntu-latest
needs: remove-duplicates
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: bot/auto-update
- name: Install Dependencies
run: npm install
- name: Sort Channels
run: node scripts/sort.js
- name: Commit Changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: '[Bot] Sort channels'
commit_user_name: iptv-bot
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
branch: bot/auto-update
file_pattern: channels/*
filter:
runs-on: ubuntu-latest
needs: sort
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: bot/auto-update
- name: Install Dependencies
run: npm install
- name: Filter Playlists
run: node scripts/filter.js
- name: Commit Changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: '[Bot] Filter channels'
commit_user_name: iptv-bot commit_user_name: iptv-bot
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
@ -310,7 +291,7 @@ jobs:
file_pattern: channels/* file_pattern: channels/*
generate: generate:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: commit-changes needs: filter
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v2
@ -376,6 +357,7 @@ jobs:
branch: bot/auto-update branch: bot/auto-update
file_pattern: README.md file_pattern: README.md
pull-request: pull-request:
if: ${{ github.ref == 'refs/heads/master' }}
needs: update-readme needs: update-readme
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
@ -408,7 +390,6 @@ jobs:
pull-request-number: ${{ steps.pr.outputs.pr_number }} pull-request-number: ${{ steps.pr.outputs.pr_number }}
merge-method: squash merge-method: squash
- name: Approve Pull Request - name: Approve Pull Request
if: github.ref == 'refs/heads/master'
uses: juliangruber/approve-pull-request-action@v1 uses: juliangruber/approve-pull-request-action@v1
with: with:
github-token: ${{ secrets.PAT }} github-token: ${{ secrets.PAT }}

View File

@ -1,257 +0,0 @@
name: clean
on:
workflow_dispatch:
schedule:
- cron: '0 6 * * 0'
jobs:
create-branch:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: ${{ github.ref }}
- name: Create Branch
uses: peterjgrainger/action-create-branch@v2.0.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
branch: 'bot/remove-broken-links'
check:
runs-on: ubuntu-latest
needs: create-branch
continue-on-error: true
strategy:
fail-fast: false
matrix:
country:
[
ad,
ae,
af,
ag,
al,
am,
an,
ao,
ar,
at,
au,
aw,
az,
ba,
bb,
bd,
be,
bf,
bg,
bh,
bn,
bo,
br,
bs,
by,
ca,
cd,
cg,
ch,
ci,
cl,
cm,
cn,
co,
cr,
cu,
cw,
cy,
cz,
de,
dk,
do,
dz,
ec,
ee,
eg,
es,
et,
fi,
fj,
fo,
fr,
pf,
ge,
gh,
gm,
gn,
gp,
gq,
gr,
gt,
hk,
hn,
hr,
ht,
hu,
id,
ie,
il,
in,
iq,
ir,
is,
it,
jm,
jo,
jp,
ke,
kg,
kh,
kp,
kr,
kw,
kz,
la,
lb,
li,
lk,
lt,
lu,
lv,
ly,
ma,
mc,
md,
me,
mk,
ml,
mm,
mn,
mo,
mt,
mv,
mx,
my,
mz,
ne,
ng,
ni,
nl,
no,
np,
nz,
om,
pa,
pe,
ph,
pk,
pl,
pr,
ps,
pt,
py,
qa,
ro,
rs,
ru,
rw,
sa,
sd,
se,
sg,
si,
sk,
sl,
sm,
sn,
so,
sv,
sy,
th,
tj,
tm,
tn,
tr,
tt,
tw,
tz,
ua,
ug,
uk,
us,
uy,
uz,
va,
ve,
vi,
vn,
xk,
ye,
zm,
unsorted
]
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: bot/remove-broken-links
- name: Setup FFmpeg
uses: FedericoCarboni/setup-ffmpeg@v1
- name: Install Dependencies
run: npm install
- name: Remove Broken Links
run: node scripts/clean.js --country=${{ matrix.country }} --debug
- name: Upload Artifact
uses: actions/upload-artifact@v2
with:
name: channels
path: channels/${{ matrix.country }}.m3u
commit-changes:
runs-on: ubuntu-latest
needs: check
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: bot/remove-broken-links
- name: Download Artifacts
uses: actions/download-artifact@v2
- name: Commit Changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: '[Bot] Remove broken links'
commit_user_name: iptv-bot
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
branch: bot/remove-broken-links
file_pattern: channels/*
pull-request:
if: ${{ github.ref == 'refs/heads/master' }}
runs-on: ubuntu-latest
needs: commit-changes
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: bot/remove-broken-links
- name: Generate Token
uses: tibdex/github-app-token@v1
id: generate-token
with:
app_id: ${{ secrets.APP_ID }}
private_key: ${{ secrets.APP_PRIVATE_KEY }}
- name: Create Pull Request
uses: repo-sync/pull-request@v2
with:
source_branch: 'bot/remove-broken-links'
destination_branch: 'master'
pr_title: '[Bot] Remove broken links'
pr_body: |
This pull request is created by [clean][1] workflow.
The script checks all links except those with labels `[Geo-blocked]`, `[Offline]` or `[Not 24/7]` in the title.
**IMPORTANT:** Before merging all links should be checked manually to make sure that the response from the server has not changed. If the link works for you but occasionally return an HTTP code 403 (Forbidden) then it should be marked as `[Geo-blocked]`. If the link does not work but has no alternative, you can mark it as `[Offline]` to save it in the playlist along with a description. Working links should be marked as `[Not 24/7]` so that the script will skip them next time.
[1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }}
pr_draft: true
github_token: ${{ steps.generate-token.outputs.token }}

View File

@ -125,7 +125,7 @@ STREAM_URL
| `LANGUAGE` | Channel language. The name of the language must conform to the standard [ISO 639-3](https://iso639-3.sil.org/code_tables/639/data?title=&field_iso639_cd_st_mmbrshp_639_1_tid=94671&name_3=&field_iso639_element_scope_tid=All&field_iso639_language_type_tid=51&items_per_page=500). If the channel is broadcast in several languages you can list them separated by a semicolon. (optional) | | `LANGUAGE` | Channel language. The name of the language must conform to the standard [ISO 639-3](https://iso639-3.sil.org/code_tables/639/data?title=&field_iso639_cd_st_mmbrshp_639_1_tid=94671&name_3=&field_iso639_element_scope_tid=All&field_iso639_language_type_tid=51&items_per_page=500). If the channel is broadcast in several languages you can list them separated by a semicolon. (optional) |
| `LOGO_URL` | The logo of the channel that will be displayed if the player supports it. Supports files in png, jpeg and gif format. (optional) | | `LOGO_URL` | The logo of the channel that will be displayed if the player supports it. Supports files in png, jpeg and gif format. (optional) |
| `CATEGORY` | The category to which the channel belongs. The list of currently supported categories can be found [here](https://github.com/iptv-org/iptv#playlists-by-category). (optional) | | `CATEGORY` | The category to which the channel belongs. The list of currently supported categories can be found [here](https://github.com/iptv-org/iptv#playlists-by-category). (optional) |
| `FULL_NAME` | Full name of the channel. It is recommended to use the name listed on [lyngsat](https://www.lyngsat.com/search.html) or [wikipedia](https://www.wikipedia.org/) if possible. May contain any characters except plus sign, minus sign, round and square brackets. | | `FULL_NAME` | Full name of the channel. It is recommended to use the name listed on [lyngsat](https://www.lyngsat.com/search.html) or [wikipedia](https://www.wikipedia.org/) if possible. May contain any characters except round and square brackets. |
| `STREAM_TIME_SHIFT` | Must be specified if the channel is broadcast with a shift in time relative to the main stream. Should only contain a number and a sign. (optional) | | `STREAM_TIME_SHIFT` | Must be specified if the channel is broadcast with a shift in time relative to the main stream. Should only contain a number and a sign. (optional) |
| `ALTERNATIVE_NAME` | Can be used to specify a short name or name in another language. May contain any characters except round and square brackets. (optional) | | `ALTERNATIVE_NAME` | Can be used to specify a short name or name in another language. May contain any characters except round and square brackets. (optional) |
| `STREAM_RESOLUTION` | The maximum height of the frame with a "p" at the end. In case of VLC Player this information can be found in `Window > Media Information... > Codec Details`. (optional) | | `STREAM_RESOLUTION` | The maximum height of the frame with a "p" at the end. In case of VLC Player this information can be found in `Window > Media Information... > Codec Details`. (optional) |
@ -153,7 +153,8 @@ http://example.com/stream.m3u8
- `.github/` - `.github/`
- `ISSUE_TEMPLATE/`: issue templates for this repository. - `ISSUE_TEMPLATE/`: issue templates for this repository.
- `workflows/` - `workflows/`
- `auto-update.yml`: contain actions that automatically updates all playlists every day. - `auto-update.yml`: GitHub Action that automatically updates all playlists every day.
- `check.yml`: GitHub Action that automatically checks every pull request for syntax errors.
- `CODE_OF_CONDUCT.md`: rules you shouldn't break if you don't want to get banned. - `CODE_OF_CONDUCT.md`: rules you shouldn't break if you don't want to get banned.
- `.readme/` - `.readme/`
- `_categories.md`: automatically generated list of all categories and their corresponding playlists. - `_categories.md`: automatically generated list of all categories and their corresponding playlists.
@ -168,8 +169,6 @@ http://example.com/stream.m3u8
- `unsorted.m3u`: playlist with channels not yet sorted. - `unsorted.m3u`: playlist with channels not yet sorted.
- `scripts/` - `scripts/`
- `helpers/`: helper scripts used in GitHub Actions. - `helpers/`: helper scripts used in GitHub Actions.
- `clean.js`: used in GitHub Action to check all links and remove broken ones.
- `detect-resolution.js`: used in GitHub Action to detect resolution of the streams.
- `filter.js`: used within GitHub Action to remove blacklisted channels from playlists. - `filter.js`: used within GitHub Action to remove blacklisted channels from playlists.
- `format.js`: used within GitHub Action to format channel descriptions. - `format.js`: used within GitHub Action to format channel descriptions.
- `generate.js`: used within GitHub Action to generate all additional playlists. - `generate.js`: used within GitHub Action to generate all additional playlists.

20
package-lock.json generated
View File

@ -7,14 +7,12 @@
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@freearhey/iso-639-3": "^1.0.0", "@freearhey/iso-639-3": "^1.0.0",
"axios": "^0.21.1",
"chalk": "^4.1.1",
"commander": "^7.0.0", "commander": "^7.0.0",
"escape-string-regexp": "^2.0.0",
"iptv-checker": "^0.20.2", "iptv-checker": "^0.20.2",
"iptv-playlist-parser": "^0.5.4", "iptv-playlist-parser": "^0.5.4",
"m3u-linter": "^0.1.3", "m3u-linter": "^0.1.3",
"markdown-include": "^0.4.3", "markdown-include": "^0.4.3",
"normalize-url": "^6.1.0",
"pre-push": "^0.1.1", "pre-push": "^0.1.1",
"progress": "^2.0.3", "progress": "^2.0.3",
"transliteration": "^2.2.0" "transliteration": "^2.2.0"
@ -2963,6 +2961,17 @@
"node": ">=0.10.0" "node": ">=0.10.0"
} }
}, },
"node_modules/normalize-url": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz",
"integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/npm-run-path": { "node_modules/npm-run-path": {
"version": "4.0.1", "version": "4.0.1",
"resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
@ -6045,6 +6054,11 @@
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
"integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="
}, },
"normalize-url": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz",
"integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A=="
},
"npm-run-path": { "npm-run-path": {
"version": "4.0.1", "version": "4.0.1",
"resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",

View File

@ -11,14 +11,12 @@
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@freearhey/iso-639-3": "^1.0.0", "@freearhey/iso-639-3": "^1.0.0",
"axios": "^0.21.1",
"chalk": "^4.1.1",
"commander": "^7.0.0", "commander": "^7.0.0",
"escape-string-regexp": "^2.0.0",
"iptv-checker": "^0.20.2", "iptv-checker": "^0.20.2",
"iptv-playlist-parser": "^0.5.4", "iptv-playlist-parser": "^0.5.4",
"m3u-linter": "^0.1.3", "m3u-linter": "^0.1.3",
"markdown-include": "^0.4.3", "markdown-include": "^0.4.3",
"normalize-url": "^6.1.0",
"pre-push": "^0.1.1", "pre-push": "^0.1.1",
"progress": "^2.0.3", "progress": "^2.0.3",
"transliteration": "^2.2.0" "transliteration": "^2.2.0"

View File

@ -1,74 +0,0 @@
const IPTVChecker = require('iptv-checker')
const { program } = require('commander')
const ProgressBar = require('progress')
const parser = require('./helpers/parser')
const utils = require('./helpers/utils')
const log = require('./helpers/log')
program
.usage('[OPTIONS]...')
.option('-d, --debug', 'Enable debug mode')
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
.option('--timeout <timeout>', 'Set timeout for each request', 5000)
.parse(process.argv)
let bar
const config = program.opts()
const ignoreStatus = ['Geo-blocked', 'Not 24/7', 'Offline']
const checker = new IPTVChecker({
timeout: config.timeout
})
async function main() {
log.start()
if (config.debug) log.print(`Debug mode enabled\n`)
let playlists = parser.parseIndex()
playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
for (const playlist of playlists) {
await parser
.parsePlaylist(playlist.url)
.then(checkPlaylist)
.then(p => p.save())
}
log.finish()
}
async function checkPlaylist(playlist) {
if (!config.debug) {
bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, {
total: playlist.channels.length
})
}
const channels = []
const total = playlist.channels.length
for (const [index, channel] of playlist.channels.entries()) {
const skipChannel =
channel.status &&
ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())
if (skipChannel) {
channels.push(channel)
} else {
const result = await checker.checkStream(channel.data)
if (result.status.ok || result.status.reason.includes('timed out')) {
channels.push(channel)
} else {
if (config.debug) log.print(`ERR: ${channel.url} (${result.status.reason})\n`)
}
}
if (!config.debug) bar.tick()
}
if (playlist.channels.length !== channels.length) {
log.print(`File '${playlist.url}' has been updated\n`)
playlist.channels = channels
playlist.updated = true
}
return playlist
}
main()

View File

@ -1,114 +0,0 @@
const { program } = require('commander')
const ProgressBar = require('progress')
const axios = require('axios')
const https = require('https')
const parser = require('./helpers/parser')
const utils = require('./helpers/utils')
const log = require('./helpers/log')
program
.usage('[OPTIONS]...')
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
.option('--delay <delay>', 'Delay between parser requests', 1000)
.option('--timeout <timeout>', 'Set timeout for each request', 5000)
.parse(process.argv)
const config = program.opts()
const ignoreStatus = ['Offline']
const instance = axios.create({
timeout: config.timeout,
maxContentLength: 200000,
httpsAgent: new https.Agent({
rejectUnauthorized: false
})
})
async function main() {
log.start()
log.print(`Parsing 'index.m3u'...\n`)
let playlists = parser.parseIndex()
playlists = utils
.filterPlaylists(playlists, config.country, config.exclude)
.filter(i => i.url !== 'channels/unsorted.m3u')
for (const playlist of playlists) {
await parser
.parsePlaylist(playlist.url)
.then(detectResolution)
.then(p => p.save())
}
log.finish()
}
async function detectResolution(playlist) {
const channels = []
const bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, {
total: playlist.channels.length
})
let updated = false
for (const channel of playlist.channels) {
bar.tick()
const skipChannel =
channel.status &&
ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())
if (!channel.resolution.height && !skipChannel) {
const CancelToken = axios.CancelToken
const source = CancelToken.source()
const timeout = setTimeout(() => {
source.cancel()
}, config.timeout)
const response = await instance
.get(channel.url, { cancelToken: source.token })
.then(res => {
clearTimeout(timeout)
return res
})
.then(utils.sleep(config.delay))
.catch(err => {
clearTimeout(timeout)
})
if (response && response.status === 200) {
if (/^#EXTM3U/.test(response.data)) {
const resolution = parseResolution(response.data)
if (resolution) {
channel.resolution = resolution
updated = true
}
}
}
}
channels.push(channel)
}
if (updated) {
log.print(`File '${playlist.url}' has been updated\n`)
playlist.channels = channels
playlist.updated = true
}
return playlist
}
function parseResolution(string) {
const regex = /RESOLUTION=(\d+)x(\d+)/gm
const match = string.matchAll(regex)
const arr = Array.from(match).map(m => ({
width: parseInt(m[1]),
height: parseInt(m[2])
}))
return arr.length
? arr.reduce(function (prev, current) {
return prev.height > current.height ? prev : current
})
: undefined
}
main()

View File

@ -1,21 +1,46 @@
const IPTVChecker = require('iptv-checker')
const normalize = require('normalize-url')
const { program } = require('commander')
const ProgressBar = require('progress')
const parser = require('./helpers/parser') const parser = require('./helpers/parser')
const utils = require('./helpers/utils') const utils = require('./helpers/utils')
const file = require('./helpers/file') const file = require('./helpers/file')
const log = require('./helpers/log') const log = require('./helpers/log')
program
.usage('[OPTIONS]...')
.option('-d, --debug', 'Enable debug mode')
.option('-s, --status', 'Update stream status')
.option('-r, --resolution', 'Detect stream resolution')
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
.option('--timeout <timeout>', 'Set timeout for each request', 5000)
.parse(process.argv)
let bar
const ignoreStatus = ['Geo-blocked', 'Not 24/7']
const config = program.opts()
const checker = new IPTVChecker({
timeout: config.timeout
})
async function main() { async function main() {
log.start() log.start()
log.print(`Parsing 'index.m3u'...`) if (config.debug) log.print(`Debug mode enabled\n`)
if (config.status) log.print(`Status check enabled\n`)
if (config.resolution) log.print(`Resolution detection enabled\n`)
let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
if (!playlists.length) log.print(`No playlist is selected\n`)
for (const playlist of playlists) { for (const playlist of playlists) {
log.print(`\nProcessing '${playlist.url}'...`)
await parser await parser
.parsePlaylist(playlist.url) .parsePlaylist(playlist.url)
.then(formatPlaylist) .then(updatePlaylist)
.then(playlist => { .then(playlist => {
if (file.read(playlist.url) !== playlist.toString()) { if (file.read(playlist.url) !== playlist.toString()) {
log.print('updated') log.print(`File '${playlist.url}' has been updated\n`)
playlist.updated = true playlist.updated = true
} }
@ -23,33 +48,115 @@ async function main() {
}) })
} }
log.print('\n')
log.finish() log.finish()
} }
async function formatPlaylist(playlist) { async function updatePlaylist(playlist) {
if (!config.debug) {
bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, {
total: playlist.channels.length
})
} else {
log.print(`Processing '${playlist.url}'...\n`)
}
for (const channel of playlist.channels) { for (const channel of playlist.channels) {
const code = file.getBasename(playlist.url) addMissingData(channel)
// add missing tvg-name updateGroupTitle(channel)
if (!channel.tvg.name && code !== 'unsorted' && channel.name) { normalizeUrl(channel)
channel.tvg.name = channel.name.replace(/\"/gi, '')
const checkOnline = config.status || config.resolution
const skipChannel =
channel.status &&
ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())
if (checkOnline && !skipChannel) {
await checker
.checkStream(channel.data)
.then(result => {
const status = parseStatus(result.status)
if (config.status) {
updateStatus(channel, status)
}
if (config.resolution && status === 'online') {
updateResolution(channel, result.status.metadata)
}
if (config.debug && status === 'offline') {
log.print(` ERR: ${channel.url} (${result.status.reason})\n`)
}
})
.catch(err => {
if (config.debug) log.print(` ERR: ${channel.url} (${err.message})\n`)
})
} }
// add missing tvg-id if (!config.debug) bar.tick()
if (!channel.tvg.id && code !== 'unsorted' && channel.tvg.name) {
const id = utils.name2id(channel.tvg.name)
channel.tvg.id = id ? `${id}.${code}` : ''
}
// add missing country
if (!channel.countries.length) {
const name = utils.code2name(code)
channel.countries = name ? [{ code, name }] : []
channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';')
}
// update group-title
channel.group.title = channel.category
} }
return playlist return playlist
} }
function parseStatus(status) {
if (status.ok) {
return 'online'
} else if (status.reason.includes('timed out')) {
return 'timeout'
} else if (status.reason.includes('403')) {
return 'error_403'
} else if (status.reason.includes('not one of 40{0,1,3,4}')) {
return 'error_40x' // 402, 451
} else {
return 'offline'
}
}
function updateStatus(channel, status) {
switch (status) {
case 'online':
channel.status = null
break
case 'offline':
channel.status = 'Offline'
break
}
}
function addMissingData(channel) {
// tvg-name
if (!channel.tvg.name && channel.name) {
channel.tvg.name = channel.name.replace(/\"/gi, '')
}
// tvg-id
if (!channel.tvg.id && channel.tvg.name) {
const id = utils.name2id(channel.tvg.name)
channel.tvg.id = id ? `${id}.${code}` : ''
}
// country
if (!channel.countries.length) {
const name = utils.code2name(code)
channel.countries = name ? [{ code, name }] : []
channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';')
}
}
function updateGroupTitle(channel) {
channel.group.title = channel.category
}
function normalizeUrl(channel) {
const normalized = normalize(channel.url, { stripWWW: false })
channel.updateUrl(normalized)
}
function updateResolution(channel, metadata) {
const streams = metadata ? metadata.streams.filter(stream => stream.codec_type === 'video') : []
if (!channel.resolution.height && streams.length) {
channel.resolution = streams.reduce((acc, curr) => {
if (curr.height > acc.height) return { width: curr.width, height: curr.height }
return acc
})
}
}
main() main()

View File

@ -22,6 +22,11 @@ module.exports = class Channel {
this.languages = this.parseLanguages(data.tvg.language) this.languages = this.parseLanguages(data.tvg.language)
} }
updateUrl(url) {
this.url = url
this.data.url = url
}
parseName(title) { parseName(title) {
return title return title
.trim() .trim()

View File

@ -1,4 +1,3 @@
const escapeStringRegexp = require('escape-string-regexp')
const transliteration = require('transliteration') const transliteration = require('transliteration')
const iso6393 = require('@freearhey/iso-639-3') const iso6393 = require('@freearhey/iso-639-3')
const categories = require('./categories') const categories = require('./categories')
@ -71,16 +70,6 @@ utils.sortBy = function (arr, fields) {
}) })
} }
utils.escapeStringRegexp = function (scring) {
return escapeStringRegexp(string)
}
utils.sleep = function (ms) {
return function (x) {
return new Promise(resolve => setTimeout(() => resolve(x), ms))
}
}
utils.removeProtocol = function (string) { utils.removeProtocol = function (string) {
return string.replace(/(^\w+:|^)\/\//, '') return string.replace(/(^\w+:|^)\/\//, '')
} }