Merge branch 'master' into LaneSh4d0w-patch-1

This commit is contained in:
Shadix A 2021-08-09 19:40:14 +02:00 committed by GitHub
commit 536418d69d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 5985 additions and 55 deletions

View File

@ -17,8 +17,9 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with: with:
branch: 'bot/remove-broken-links' branch: 'bot/remove-broken-links'
test: check:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: create-branch
continue-on-error: true continue-on-error: true
strategy: strategy:
fail-fast: false fail-fast: false
@ -193,6 +194,8 @@ jobs:
uses: actions/checkout@v2 uses: actions/checkout@v2
with: with:
ref: bot/remove-broken-links ref: bot/remove-broken-links
- name: Setup FFmpeg
uses: FedericoCarboni/setup-ffmpeg@v1
- name: Install Dependencies - name: Install Dependencies
run: npm install run: npm install
- name: Remove Broken Links - name: Remove Broken Links
@ -204,7 +207,7 @@ jobs:
path: channels/${{ matrix.country }}.m3u path: channels/${{ matrix.country }}.m3u
commit-changes: commit-changes:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: test needs: check
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v2
@ -215,13 +218,14 @@ jobs:
- name: Commit Changes - name: Commit Changes
uses: stefanzweifel/git-auto-commit-action@v4 uses: stefanzweifel/git-auto-commit-action@v4
with: with:
commit_message: '[Bot] Update playlists' commit_message: '[Bot] Remove broken links'
commit_user_name: iptv-bot commit_user_name: iptv-bot
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
branch: bot/remove-broken-links branch: bot/remove-broken-links
file_pattern: channels/* file_pattern: channels/*
pull-request: pull-request:
if: ${{ github.ref == 'refs/heads/master' }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: commit-changes needs: commit-changes
steps: steps:
@ -244,9 +248,9 @@ jobs:
pr_body: | pr_body: |
This pull request is created by [clean][1] workflow. This pull request is created by [clean][1] workflow.
The script checks each link and removes only those that return a HTTP 404 code (Not Found). Also, the script ignores links with labels `[Geo-blocked]` and `[Not 24/7]` in the title. The script checks all links except those with labels `[Geo-blocked]`, `[Offline]` or `[Not 24/7]` in the title.
**IMPORTANT:** Before merging all links should be checked manually to make sure that the response from the server has not changed. Working links should be marked as `[Not 24/7]` so that next time the script will not delete them. **IMPORTANT:** Before merging all links should be checked manually to make sure that the response from the server has not changed. If the link works for you but occasionally return an HTTP code 403 (Forbidden) then it should be marked as `[Geo-blocked]`. If the link does not work but has no alternative, you can mark it as `[Offline]` to save it in the playlist along with a description. Working links should be marked as `[Not 24/7]` so that the script will skip them next time.
[1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }} [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }}
pr_draft: true pr_draft: true

5952
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -15,6 +15,7 @@
"chalk": "^4.1.1", "chalk": "^4.1.1",
"commander": "^7.0.0", "commander": "^7.0.0",
"escape-string-regexp": "^2.0.0", "escape-string-regexp": "^2.0.0",
"iptv-checker": "^0.20.2",
"iptv-playlist-parser": "^0.5.4", "iptv-playlist-parser": "^0.5.4",
"m3u-linter": "^0.1.3", "m3u-linter": "^0.1.3",
"markdown-include": "^0.4.3", "markdown-include": "^0.4.3",

View File

@ -1,93 +1,65 @@
const IPTVChecker = require('iptv-checker')
const { program } = require('commander') const { program } = require('commander')
const ProgressBar = require('progress') const ProgressBar = require('progress')
const axios = require('axios')
const https = require('https')
const chalk = require('chalk')
const parser = require('./helpers/parser') const parser = require('./helpers/parser')
const utils = require('./helpers/utils') const utils = require('./helpers/utils')
const log = require('./helpers/log') const log = require('./helpers/log')
program program
.usage('[OPTIONS]...') .usage('[OPTIONS]...')
.option('-d, --debug', 'Enable debug mode')
.option('-c, --country <country>', 'Comma-separated list of country codes', '') .option('-c, --country <country>', 'Comma-separated list of country codes', '')
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '') .option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
.option('--delay <delay>', 'Delay between parser requests', 1000)
.option('--timeout <timeout>', 'Set timeout for each request', 5000) .option('--timeout <timeout>', 'Set timeout for each request', 5000)
.parse(process.argv) .parse(process.argv)
let bar
const config = program.opts() const config = program.opts()
const offlineStatusCodes = [404, 410, 451, 500, 501]
const ignoreStatus = ['Geo-blocked', 'Not 24/7', 'Offline'] const ignoreStatus = ['Geo-blocked', 'Not 24/7', 'Offline']
const instance = axios.create({ const checker = new IPTVChecker({
timeout: config.timeout, timeout: config.timeout
maxContentLength: 200000,
httpsAgent: new https.Agent({
rejectUnauthorized: false
}),
validateStatus: function (status) {
return !offlineStatusCodes.includes(status)
}
}) })
let broken = 0
async function main() { async function main() {
log.start() log.start()
log.print(`Parsing 'index.m3u'...`) if (config.debug) log.print(`Debug mode enabled\n`)
let playlists = parser.parseIndex() let playlists = parser.parseIndex()
playlists = utils.filterPlaylists(playlists, config.country, config.exclude) playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
for (const playlist of playlists) { for (const playlist of playlists) {
await parser await parser
.parsePlaylist(playlist.url) .parsePlaylist(playlist.url)
.then(checkStatus) .then(checkPlaylist)
.then(p => p.save()) .then(p => p.save())
} }
log.finish() log.finish()
} }
async function checkStatus(playlist) { async function checkPlaylist(playlist) {
let bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, { if (!config.debug) {
total: playlist.channels.length bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, {
}) total: playlist.channels.length
})
}
const channels = [] const channels = []
const total = playlist.channels.length const total = playlist.channels.length
for (const [index, channel] of playlist.channels.entries()) { for (const [index, channel] of playlist.channels.entries()) {
const current = index + 1
const counter = chalk.gray(`[${current}/${total}]`)
const skipChannel = const skipChannel =
channel.status && channel.status &&
ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())
bar.tick() if (skipChannel) {
if (
skipChannel ||
(!channel.url.startsWith('http://') && !channel.url.startsWith('https://'))
) {
channels.push(channel) channels.push(channel)
} else { } else {
const CancelToken = axios.CancelToken const result = await checker.checkStream(channel.data)
const source = CancelToken.source() if (result.status.ok || result.status.reason.includes('timed out')) {
const timeout = setTimeout(() => { channels.push(channel)
source.cancel() } else {
}, config.timeout) if (config.debug) log.print(`ERR: ${channel.url} (${result.status.reason})\n`)
}
await instance
.get(channel.url, { cancelToken: source.token })
.then(() => {
clearTimeout(timeout)
channels.push(channel)
})
.then(utils.sleep(config.delay))
.catch(err => {
clearTimeout(timeout)
if (err.response && offlineStatusCodes.includes(err.response.status)) {
broken++
} else {
channels.push(channel)
}
})
} }
if (!config.debug) bar.tick()
} }
if (playlist.channels.length !== channels.length) { if (playlist.channels.length !== channels.length) {

View File

@ -7,6 +7,7 @@ const nsfwCategories = categories.filter(c => c.nsfw).map(c => c.name)
module.exports = class Channel { module.exports = class Channel {
constructor(data) { constructor(data) {
this.data = data
this.raw = data.raw this.raw = data.raw
this.tvg = data.tvg this.tvg = data.tvg
this.http = data.http this.http = data.http