mirror of
https://github.com/iptv-org/iptv.git
synced 2024-09-06 00:02:24 +02:00
Merge branch 'master' into LaneSh4d0w-clean-ru
This commit is contained in:
Dum4G
GitHub
commit
a9b34c6529
400
.github/workflows/auto-update.yml
vendored
400
.github/workflows/auto-update.yml
vendored
@ -4,103 +4,382 @@ on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *'
|
||||
jobs:
|
||||
remove-duplicates:
|
||||
create-branch:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
- name: Remove Duplicates
|
||||
run: node scripts/remove-duplicates.js
|
||||
- name: Upload Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: channels
|
||||
path: channels/
|
||||
filter:
|
||||
runs-on: ubuntu-latest
|
||||
needs: remove-duplicates
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v2
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
- name: Filter Playlists
|
||||
run: node scripts/filter.js
|
||||
- name: Upload Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
ref: ${{ github.ref }}
|
||||
- name: Create Branch
|
||||
uses: peterjgrainger/action-create-branch@v2.0.1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
name: channels
|
||||
path: channels/
|
||||
branch: 'bot/auto-update'
|
||||
format:
|
||||
runs-on: ubuntu-latest
|
||||
needs: filter
|
||||
needs: create-branch
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
ref: bot/auto-update
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
- name: Format Playlists
|
||||
run: node scripts/format.js
|
||||
- name: Upload Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
- name: Commit Changes
|
||||
uses: stefanzweifel/git-auto-commit-action@v4
|
||||
with:
|
||||
name: channels
|
||||
path: channels/
|
||||
generate:
|
||||
commit_message: '[Bot] Formate playlists'
|
||||
commit_user_name: iptv-bot
|
||||
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
|
||||
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
|
||||
branch: bot/auto-update
|
||||
file_pattern: channels/*
|
||||
remove-duplicates:
|
||||
runs-on: ubuntu-latest
|
||||
needs: format
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
ref: bot/auto-update
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
- name: Remove Duplicates
|
||||
run: node scripts/remove-duplicates.js
|
||||
- name: Commit Changes
|
||||
uses: stefanzweifel/git-auto-commit-action@v4
|
||||
with:
|
||||
commit_message: '[Bot] Remove duplicates'
|
||||
commit_user_name: iptv-bot
|
||||
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
|
||||
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
|
||||
branch: bot/auto-update
|
||||
file_pattern: channels/*
|
||||
sort:
|
||||
runs-on: ubuntu-latest
|
||||
needs: remove-duplicates
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
ref: bot/auto-update
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
- name: Sort Channels
|
||||
run: node scripts/sort.js
|
||||
- name: Commit Changes
|
||||
uses: stefanzweifel/git-auto-commit-action@v4
|
||||
with:
|
||||
commit_message: '[Bot] Sort channels'
|
||||
commit_user_name: iptv-bot
|
||||
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
|
||||
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
|
||||
branch: bot/auto-update
|
||||
file_pattern: channels/*
|
||||
filter:
|
||||
runs-on: ubuntu-latest
|
||||
needs: sort
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
ref: bot/auto-update
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
- name: Filter Playlists
|
||||
run: node scripts/filter.js
|
||||
- name: Commit Changes
|
||||
uses: stefanzweifel/git-auto-commit-action@v4
|
||||
with:
|
||||
commit_message: '[Bot] Filter channels'
|
||||
commit_user_name: iptv-bot
|
||||
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
|
||||
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
|
||||
branch: bot/auto-update
|
||||
file_pattern: channels/*
|
||||
detect-resolution:
|
||||
runs-on: ubuntu-latest
|
||||
needs: filter
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
country:
|
||||
[
|
||||
ad,
|
||||
ae,
|
||||
af,
|
||||
ag,
|
||||
al,
|
||||
am,
|
||||
an,
|
||||
ao,
|
||||
ar,
|
||||
at,
|
||||
au,
|
||||
aw,
|
||||
az,
|
||||
ba,
|
||||
bb,
|
||||
bd,
|
||||
be,
|
||||
bf,
|
||||
bg,
|
||||
bh,
|
||||
bn,
|
||||
bo,
|
||||
br,
|
||||
bs,
|
||||
by,
|
||||
ca,
|
||||
cd,
|
||||
cg,
|
||||
ch,
|
||||
ci,
|
||||
cl,
|
||||
cm,
|
||||
cn,
|
||||
co,
|
||||
cr,
|
||||
cu,
|
||||
cw,
|
||||
cy,
|
||||
cz,
|
||||
de,
|
||||
dk,
|
||||
do,
|
||||
dz,
|
||||
ec,
|
||||
ee,
|
||||
eg,
|
||||
es,
|
||||
et,
|
||||
fi,
|
||||
fj,
|
||||
fo,
|
||||
fr,
|
||||
ge,
|
||||
gh,
|
||||
gm,
|
||||
gn,
|
||||
gp,
|
||||
gq,
|
||||
gr,
|
||||
gt,
|
||||
hk,
|
||||
hn,
|
||||
hr,
|
||||
ht,
|
||||
hu,
|
||||
id,
|
||||
ie,
|
||||
il,
|
||||
in,
|
||||
iq,
|
||||
ir,
|
||||
is,
|
||||
it,
|
||||
jm,
|
||||
jo,
|
||||
jp,
|
||||
ke,
|
||||
kg,
|
||||
kh,
|
||||
kp,
|
||||
kr,
|
||||
kw,
|
||||
kz,
|
||||
la,
|
||||
lb,
|
||||
li,
|
||||
lk,
|
||||
lt,
|
||||
lu,
|
||||
lv,
|
||||
ly,
|
||||
ma,
|
||||
mc,
|
||||
md,
|
||||
me,
|
||||
mk,
|
||||
ml,
|
||||
mm,
|
||||
mn,
|
||||
mo,
|
||||
mt,
|
||||
mv,
|
||||
mx,
|
||||
my,
|
||||
mz,
|
||||
ne,
|
||||
ng,
|
||||
ni,
|
||||
nl,
|
||||
no,
|
||||
np,
|
||||
nz,
|
||||
om,
|
||||
pa,
|
||||
pe,
|
||||
ph,
|
||||
pk,
|
||||
pl,
|
||||
pr,
|
||||
ps,
|
||||
pt,
|
||||
py,
|
||||
qa,
|
||||
ro,
|
||||
rs,
|
||||
ru,
|
||||
rw,
|
||||
sa,
|
||||
sd,
|
||||
se,
|
||||
sg,
|
||||
si,
|
||||
sk,
|
||||
sl,
|
||||
sm,
|
||||
sn,
|
||||
so,
|
||||
sv,
|
||||
sy,
|
||||
th,
|
||||
tj,
|
||||
tm,
|
||||
tn,
|
||||
tr,
|
||||
tt,
|
||||
tw,
|
||||
tz,
|
||||
ua,
|
||||
ug,
|
||||
uk,
|
||||
us,
|
||||
uy,
|
||||
uz,
|
||||
va,
|
||||
ve,
|
||||
vi,
|
||||
vn,
|
||||
xk,
|
||||
ye,
|
||||
zm
|
||||
]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
ref: bot/auto-update
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
- name: Detect Resolution
|
||||
run: node scripts/detect-resolution.js --country=${{ matrix.country }}
|
||||
- name: Upload Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: channels
|
||||
path: channels/${{ matrix.country }}.m3u
|
||||
commit-changes:
|
||||
runs-on: ubuntu-latest
|
||||
needs: detect-resolution
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
ref: bot/auto-update
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: channels
|
||||
- name: Commit Changes
|
||||
uses: stefanzweifel/git-auto-commit-action@v4
|
||||
with:
|
||||
commit_message: '[Bot] Detect resolution'
|
||||
commit_user_name: iptv-bot
|
||||
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
|
||||
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
|
||||
branch: bot/auto-update
|
||||
file_pattern: channels/*
|
||||
generate:
|
||||
runs-on: ubuntu-latest
|
||||
needs: commit-changes
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
ref: bot/auto-update
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
- name: Generate Playlists
|
||||
run: node scripts/generate.js
|
||||
- name: Upload Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: gh-pages
|
||||
path: .gh-pages/
|
||||
deploy:
|
||||
if: ${{ github.ref == 'refs/heads/master' }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: generate
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
ref: bot/auto-update
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: gh-pages
|
||||
- name: Generate Token
|
||||
uses: tibdex/github-app-token@v1
|
||||
id: generate-token
|
||||
with:
|
||||
app_id: ${{ secrets.APP_ID }}
|
||||
private_key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
- name: Deploy to GitHub Pages
|
||||
uses: JamesIves/github-pages-deploy-action@4.1.1
|
||||
with:
|
||||
branch: gh-pages
|
||||
folder: .gh-pages
|
||||
folder: gh-pages
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
git-config-name: iptv-bot
|
||||
git-config-email: 84861620+iptv-bot[bot]@users.noreply.github.com
|
||||
commit-message: '[Bot] Deploy to GitHub Pages'
|
||||
update-readme:
|
||||
runs-on: ubuntu-latest
|
||||
needs: generate
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
ref: bot/auto-update
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
- name: Update README.md
|
||||
run: node scripts/update-readme.js
|
||||
- name: Upload Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
- name: Commit Changes
|
||||
uses: stefanzweifel/git-auto-commit-action@v4
|
||||
with:
|
||||
name: README.md
|
||||
path: README.md
|
||||
commit_message: '[Bot] Update README.md'
|
||||
commit_user_name: iptv-bot
|
||||
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
|
||||
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
|
||||
branch: bot/auto-update
|
||||
file_pattern: README.md
|
||||
pull-request:
|
||||
needs: update-readme
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
- name: Download /channels
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: channels
|
||||
path: channels/
|
||||
- name: Download README.md
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: README.md
|
||||
ref: bot/auto-update
|
||||
- name: Generate Token
|
||||
uses: tibdex/github-app-token@v1
|
||||
id: generate-token
|
||||
@ -109,26 +388,25 @@ jobs:
|
||||
private_key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
- name: Create Pull Request
|
||||
id: pr
|
||||
uses: peter-evans/create-pull-request@v3
|
||||
uses: repo-sync/pull-request@v2
|
||||
with:
|
||||
title: '[Bot] Update playlists'
|
||||
body: |
|
||||
This pull request is created automatically by `auto-update` action.
|
||||
commit-message: '[Bot] Update playlists'
|
||||
committer: GitHub <noreply@github.com>
|
||||
branch: bot/auto-update
|
||||
delete-branch: true
|
||||
token: ${{ steps.generate-token.outputs.token }}
|
||||
source_branch: 'bot/auto-update'
|
||||
destination_branch: 'master'
|
||||
pr_title: '[Bot] Update playlists'
|
||||
pr_body: |
|
||||
This pull request is created by [auto-update][1] workflow.
|
||||
|
||||
[1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }}
|
||||
github_token: ${{ steps.generate-token.outputs.token }}
|
||||
- name: Enable Pull Request Automerge
|
||||
if: steps.pr.outputs.pull-request-operation == 'created'
|
||||
uses: peter-evans/enable-pull-request-automerge@v1
|
||||
with:
|
||||
token: ${{ secrets.PAT }}
|
||||
pull-request-number: ${{ steps.pr.outputs.pull-request-number }}
|
||||
pull-request-number: ${{ steps.pr.outputs.pr_number }}
|
||||
merge-method: squash
|
||||
- name: Approve Pull Request
|
||||
if: steps.pr.outputs.pull-request-operation == 'created'
|
||||
if: github.ref == 'refs/heads/master'
|
||||
uses: juliangruber/approve-pull-request-action@v1
|
||||
with:
|
||||
github-token: ${{ secrets.PAT }}
|
||||
number: ${{ steps.pr.outputs.pull-request-number }}
|
||||
number: ${{ steps.pr.outputs.pr_number }}
|
||||
|
||||
@ -167,18 +167,15 @@ http://example.com/stream.m3u8
|
||||
- ...
|
||||
- `unsorted.m3u`: playlist with channels not yet sorted.
|
||||
- `scripts/`
|
||||
- `blacklist.json`: list of channels banned for addition to the repository.
|
||||
- `categories.json`: list of supported categories.
|
||||
- `helpers/`: helper scripts used in GitHub Actions.
|
||||
- `clean.js`: used in GitHub Action to check all links and remove broken ones.
|
||||
- `db.js`: contains functions for retrieving and managing the channel list.
|
||||
- `detect-resolution.js`: used in GitHub Action to detect resolution of the streams.
|
||||
- `filter.js`: used within GitHub Action to remove blacklisted channels from playlists.
|
||||
- `format.js`: used within GitHub Action to format channel descriptions and sort playlists.
|
||||
- `format.js`: used within GitHub Action to format channel descriptions.
|
||||
- `generate.js`: used within GitHub Action to generate all additional playlists.
|
||||
- `parser.js`: contains functions for parsing playlists.
|
||||
- `regions.json`: list of supported region codes.
|
||||
- `remove-duplicates.js`: used in GitHub Action to remove duplicates from the playlist.
|
||||
- `sort.js`: used within GitHub Action to sort channels by name.
|
||||
- `update-readme.js`: used within GitHub Action to update the `README.md` file.
|
||||
- `utils.js`: contains functions that are used in other scripts.
|
||||
- `CONTRIBUTING.md`: file you are currently reading.
|
||||
- `index.m3u`: main playlist that contains links to all playlists in the `channels/` folder.
|
||||
- `README.md`: project description generated from the contents of the `.readme/` folder.
|
||||
|
||||
107
scripts/clean.js
107
scripts/clean.js
@ -1,14 +1,14 @@
|
||||
const { program } = require('commander')
|
||||
const parser = require('./parser')
|
||||
const utils = require('./utils')
|
||||
const axios = require('axios')
|
||||
const ProgressBar = require('progress')
|
||||
const axios = require('axios')
|
||||
const https = require('https')
|
||||
const chalk = require('chalk')
|
||||
const parser = require('./helpers/parser')
|
||||
const utils = require('./helpers/utils')
|
||||
const log = require('./helpers/log')
|
||||
|
||||
program
|
||||
.usage('[OPTIONS]...')
|
||||
.option('-d, --debug', 'Debug mode')
|
||||
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
|
||||
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
|
||||
.option('--delay <delay>', 'Delay between parser requests', 1000)
|
||||
@ -16,8 +16,8 @@ program
|
||||
.parse(process.argv)
|
||||
|
||||
const config = program.opts()
|
||||
|
||||
const offlineStatusCodes = [404, 410, 500, 501]
|
||||
const ignore = ['Geo-blocked', 'Not 24/7']
|
||||
const instance = axios.create({
|
||||
timeout: config.timeout,
|
||||
maxContentLength: 200000,
|
||||
@ -29,60 +29,39 @@ const instance = axios.create({
|
||||
}
|
||||
})
|
||||
|
||||
const ignore = ['Geo-blocked', 'Not 24/7']
|
||||
|
||||
const stats = { broken: 0 }
|
||||
let broken = 0
|
||||
|
||||
async function main() {
|
||||
console.info(`\nStarting...`)
|
||||
console.time('Done in')
|
||||
if (config.debug) {
|
||||
console.info(chalk.yellow(`INFO: Debug mode enabled\n`))
|
||||
}
|
||||
const playlists = parseIndex()
|
||||
log.start()
|
||||
|
||||
for (const playlist of playlists) {
|
||||
await loadPlaylist(playlist.url).then(checkStatus).then(savePlaylist).then(done)
|
||||
}
|
||||
|
||||
finish()
|
||||
}
|
||||
|
||||
function parseIndex() {
|
||||
console.info(`Parsing 'index.m3u'...`)
|
||||
log.print(`Parsing 'index.m3u'...`)
|
||||
let playlists = parser.parseIndex()
|
||||
playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
|
||||
console.info(`Found ${playlists.length} playlist(s)\n`)
|
||||
for (const playlist of playlists) {
|
||||
await parser
|
||||
.parsePlaylist(playlist.url)
|
||||
.then(checkStatus)
|
||||
.then(p => p.save())
|
||||
}
|
||||
|
||||
return playlists
|
||||
}
|
||||
|
||||
async function loadPlaylist(url) {
|
||||
console.info(`Processing '${url}'...`)
|
||||
return parser.parsePlaylist(url)
|
||||
log.finish()
|
||||
}
|
||||
|
||||
async function checkStatus(playlist) {
|
||||
let bar
|
||||
if (!config.debug) {
|
||||
bar = new ProgressBar(' Testing: [:bar] :current/:total (:percent) ', {
|
||||
total: playlist.channels.length
|
||||
})
|
||||
}
|
||||
const results = []
|
||||
let bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, {
|
||||
total: playlist.channels.length
|
||||
})
|
||||
const channels = []
|
||||
const total = playlist.channels.length
|
||||
for (const [index, channel] of playlist.channels.entries()) {
|
||||
const current = index + 1
|
||||
const counter = chalk.gray(`[${current}/${total}]`)
|
||||
if (bar) bar.tick()
|
||||
bar.tick()
|
||||
if (
|
||||
(channel.status && ignore.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())) ||
|
||||
(!channel.url.startsWith('http://') && !channel.url.startsWith('https://'))
|
||||
) {
|
||||
results.push(channel)
|
||||
if (config.debug) {
|
||||
console.info(` ${counter} ${chalk.green('online')} ${chalk.white(channel.url)}`)
|
||||
}
|
||||
channels.push(channel)
|
||||
} else {
|
||||
const CancelToken = axios.CancelToken
|
||||
const source = CancelToken.source()
|
||||
@ -94,55 +73,27 @@ async function checkStatus(playlist) {
|
||||
.get(channel.url, { cancelToken: source.token })
|
||||
.then(() => {
|
||||
clearTimeout(timeout)
|
||||
results.push(channel)
|
||||
if (config.debug) {
|
||||
console.info(` ${counter} ${chalk.green('online')} ${chalk.white(channel.url)}`)
|
||||
}
|
||||
channels.push(channel)
|
||||
})
|
||||
.then(utils.sleep(config.delay))
|
||||
.catch(err => {
|
||||
clearTimeout(timeout)
|
||||
if (err.response && offlineStatusCodes.includes(err.response.status)) {
|
||||
if (config.debug) {
|
||||
console.info(` ${counter} ${chalk.red('offline')} ${chalk.white(channel.url)}`)
|
||||
}
|
||||
stats.broken++
|
||||
broken++
|
||||
} else {
|
||||
results.push(channel)
|
||||
if (config.debug) {
|
||||
console.info(` ${counter} ${chalk.green('online')} ${chalk.white(channel.url)}`)
|
||||
}
|
||||
channels.push(channel)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
playlist.channels = results
|
||||
if (playlist.channels.length !== channels.length) {
|
||||
log.print(`File '${playlist.url}' has been updated\n`)
|
||||
playlist.channels = channels
|
||||
playlist.updated = true
|
||||
}
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function savePlaylist(playlist) {
|
||||
const original = utils.readFile(playlist.url)
|
||||
const output = playlist.toString({ raw: true })
|
||||
|
||||
if (original === output) {
|
||||
console.info(`No changes have been made.`)
|
||||
return false
|
||||
} else {
|
||||
utils.createFile(playlist.url, output)
|
||||
console.info(`Playlist has been updated. Removed ${stats.broken} links.`)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
async function done() {
|
||||
console.info(` `)
|
||||
}
|
||||
|
||||
function finish() {
|
||||
console.timeEnd('Done in')
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
110
scripts/detect-resolution.js
Normal file
110
scripts/detect-resolution.js
Normal file
@ -0,0 +1,110 @@
|
||||
const { program } = require('commander')
|
||||
const ProgressBar = require('progress')
|
||||
const axios = require('axios')
|
||||
const https = require('https')
|
||||
const parser = require('./helpers/parser')
|
||||
const utils = require('./helpers/utils')
|
||||
const log = require('./helpers/log')
|
||||
|
||||
program
|
||||
.usage('[OPTIONS]...')
|
||||
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
|
||||
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
|
||||
.option('--delay <delay>', 'Delay between parser requests', 1000)
|
||||
.option('--timeout <timeout>', 'Set timeout for each request', 5000)
|
||||
.parse(process.argv)
|
||||
|
||||
const config = program.opts()
|
||||
const instance = axios.create({
|
||||
timeout: config.timeout,
|
||||
maxContentLength: 200000,
|
||||
httpsAgent: new https.Agent({
|
||||
rejectUnauthorized: false
|
||||
})
|
||||
})
|
||||
|
||||
async function main() {
|
||||
log.start()
|
||||
|
||||
log.print(`Parsing 'index.m3u'...\n`)
|
||||
let playlists = parser.parseIndex()
|
||||
playlists = utils
|
||||
.filterPlaylists(playlists, config.country, config.exclude)
|
||||
.filter(i => i.url !== 'channels/unsorted.m3u')
|
||||
|
||||
for (const playlist of playlists) {
|
||||
await parser
|
||||
.parsePlaylist(playlist.url)
|
||||
.then(detectResolution)
|
||||
.then(p => p.save())
|
||||
}
|
||||
|
||||
log.finish()
|
||||
}
|
||||
|
||||
async function detectResolution(playlist) {
|
||||
const channels = []
|
||||
const bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, {
|
||||
total: playlist.channels.length
|
||||
})
|
||||
let updated = false
|
||||
for (const channel of playlist.channels) {
|
||||
bar.tick()
|
||||
if (!channel.resolution.height) {
|
||||
const CancelToken = axios.CancelToken
|
||||
const source = CancelToken.source()
|
||||
const timeout = setTimeout(() => {
|
||||
source.cancel()
|
||||
}, config.timeout)
|
||||
|
||||
const response = await instance
|
||||
.get(channel.url, { cancelToken: source.token })
|
||||
.then(res => {
|
||||
clearTimeout(timeout)
|
||||
|
||||
return res
|
||||
})
|
||||
.then(utils.sleep(config.delay))
|
||||
.catch(err => {
|
||||
clearTimeout(timeout)
|
||||
})
|
||||
|
||||
if (response && response.status === 200) {
|
||||
if (/^#EXTM3U/.test(response.data)) {
|
||||
const resolution = parseResolution(response.data)
|
||||
if (resolution) {
|
||||
channel.resolution = resolution
|
||||
updated = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
channels.push(channel)
|
||||
}
|
||||
|
||||
if (updated) {
|
||||
log.print(`File '${playlist.url}' has been updated\n`)
|
||||
playlist.channels = channels
|
||||
playlist.updated = true
|
||||
}
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
function parseResolution(string) {
|
||||
const regex = /RESOLUTION=(\d+)x(\d+)/gm
|
||||
const match = string.matchAll(regex)
|
||||
const arr = Array.from(match).map(m => ({
|
||||
width: parseInt(m[1]),
|
||||
height: parseInt(m[2])
|
||||
}))
|
||||
|
||||
return arr.length
|
||||
? arr.reduce(function (prev, current) {
|
||||
return prev.height > current.height ? prev : current
|
||||
})
|
||||
: undefined
|
||||
}
|
||||
|
||||
main()
|
||||
@ -1,67 +1,43 @@
|
||||
const parser = require('./parser')
|
||||
const utils = require('./utils')
|
||||
const blacklist = require('./blacklist.json')
|
||||
const blacklist = require('./helpers/blacklist.json')
|
||||
const parser = require('./helpers/parser')
|
||||
const log = require('./helpers/log')
|
||||
|
||||
async function main() {
|
||||
const playlists = parseIndex()
|
||||
log.start()
|
||||
|
||||
log.print(`Parsing 'index.m3u'...`)
|
||||
const playlists = parser.parseIndex()
|
||||
for (const playlist of playlists) {
|
||||
await loadPlaylist(playlist.url).then(removeBlacklisted).then(savePlaylist).then(done)
|
||||
log.print(`\nProcessing '${playlist.url}'...`)
|
||||
await parser
|
||||
.parsePlaylist(playlist.url)
|
||||
.then(removeBlacklisted)
|
||||
.then(p => p.save())
|
||||
}
|
||||
|
||||
finish()
|
||||
}
|
||||
|
||||
function parseIndex() {
|
||||
console.info(`Parsing 'index.m3u'...`)
|
||||
let playlists = parser.parseIndex()
|
||||
console.info(`Found ${playlists.length} playlist(s)\n`)
|
||||
|
||||
return playlists
|
||||
}
|
||||
|
||||
async function loadPlaylist(url) {
|
||||
console.info(`Processing '${url}'...`)
|
||||
return parser.parsePlaylist(url)
|
||||
log.print('\n')
|
||||
log.finish()
|
||||
}
|
||||
|
||||
async function removeBlacklisted(playlist) {
|
||||
console.info(` Looking for blacklisted channels...`)
|
||||
playlist.channels = playlist.channels.filter(channel => {
|
||||
return !blacklist.find(i => {
|
||||
const channelName = channel.name.toLowerCase()
|
||||
return (
|
||||
(i.name.toLowerCase() === channelName ||
|
||||
i.aliases.map(i => i.toLowerCase()).includes(channelName)) &&
|
||||
i.country === channel.filename
|
||||
)
|
||||
const channels = playlist.channels.filter(channel => {
|
||||
return !blacklist.find(item => {
|
||||
const hasSameName =
|
||||
item.name.toLowerCase() === channel.name.toLowerCase() ||
|
||||
item.aliases.map(alias => alias.toLowerCase()).includes(channel.name.toLowerCase())
|
||||
const fromSameCountry = channel.countries.find(c => c.code === item.country)
|
||||
|
||||
return hasSameName && fromSameCountry
|
||||
})
|
||||
})
|
||||
|
||||
if (playlist.channels.length !== channels.length) {
|
||||
log.print(`updated`)
|
||||
playlist.channels = channels
|
||||
playlist.updated = true
|
||||
}
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function savePlaylist(playlist) {
|
||||
console.info(` Saving playlist...`)
|
||||
const original = utils.readFile(playlist.url)
|
||||
const output = playlist.toString({ raw: true })
|
||||
|
||||
if (original === output) {
|
||||
console.info(`No changes have been made.`)
|
||||
return false
|
||||
} else {
|
||||
utils.createFile(playlist.url, output)
|
||||
console.info(`Playlist has been updated.`)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
async function done() {
|
||||
console.info(` `)
|
||||
}
|
||||
|
||||
function finish() {
|
||||
console.info('Done.')
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
@ -1,152 +1,55 @@
|
||||
const { program } = require('commander')
|
||||
const parser = require('./parser')
|
||||
const utils = require('./utils')
|
||||
const axios = require('axios')
|
||||
const ProgressBar = require('progress')
|
||||
const https = require('https')
|
||||
|
||||
program
|
||||
.usage('[OPTIONS]...')
|
||||
.option('-d, --debug', 'Debug mode')
|
||||
.option('-r --resolution', 'Parse stream resolution')
|
||||
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
|
||||
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
|
||||
.option('--delay <delay>', 'Delay between parser requests', 1000)
|
||||
.option('--timeout <timeout>', 'Set timeout for each request', 5000)
|
||||
.parse(process.argv)
|
||||
|
||||
const config = program.opts()
|
||||
|
||||
const instance = axios.create({
|
||||
timeout: config.timeout,
|
||||
maxContentLength: 200000,
|
||||
httpsAgent: new https.Agent({
|
||||
rejectUnauthorized: false
|
||||
})
|
||||
})
|
||||
const parser = require('./helpers/parser')
|
||||
const utils = require('./helpers/utils')
|
||||
const file = require('./helpers/file')
|
||||
const log = require('./helpers/log')
|
||||
|
||||
async function main() {
|
||||
console.info('Starting...')
|
||||
console.time('Done in')
|
||||
|
||||
const playlists = parseIndex()
|
||||
log.start()
|
||||
|
||||
log.print(`Parsing 'index.m3u'...`)
|
||||
let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
|
||||
for (const playlist of playlists) {
|
||||
await loadPlaylist(playlist.url)
|
||||
.then(sortChannels)
|
||||
.then(detectResolution)
|
||||
.then(savePlaylist)
|
||||
.then(done)
|
||||
}
|
||||
|
||||
finish()
|
||||
}
|
||||
|
||||
function parseIndex() {
|
||||
console.info(`\nParsing 'index.m3u'...`)
|
||||
let playlists = parser.parseIndex()
|
||||
playlists = utils
|
||||
.filterPlaylists(playlists, config.country, config.exclude)
|
||||
.filter(i => i.url !== 'channels/unsorted.m3u')
|
||||
console.info(`Found ${playlists.length} playlist(s)\n`)
|
||||
|
||||
return playlists
|
||||
}
|
||||
|
||||
async function loadPlaylist(url) {
|
||||
console.info(`Processing '${url}'...`)
|
||||
return parser.parsePlaylist(url)
|
||||
}
|
||||
|
||||
async function sortChannels(playlist) {
|
||||
console.info(` Sorting channels...`)
|
||||
playlist.channels = utils.sortBy(playlist.channels, ['name', 'url'])
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function detectResolution(playlist) {
|
||||
if (!config.resolution) return playlist
|
||||
console.log(' Detecting resolution...')
|
||||
const bar = new ProgressBar(' Progress: [:bar] :current/:total (:percent) ', {
|
||||
total: playlist.channels.length
|
||||
})
|
||||
const results = []
|
||||
for (const channel of playlist.channels) {
|
||||
bar.tick()
|
||||
if (!channel.resolution.height) {
|
||||
const CancelToken = axios.CancelToken
|
||||
const source = CancelToken.source()
|
||||
const timeout = setTimeout(() => {
|
||||
source.cancel()
|
||||
}, config.timeout)
|
||||
|
||||
const response = await instance
|
||||
.get(channel.url, { cancelToken: source.token })
|
||||
.then(res => {
|
||||
clearTimeout(timeout)
|
||||
|
||||
return res
|
||||
})
|
||||
.then(utils.sleep(config.delay))
|
||||
.catch(err => {
|
||||
clearTimeout(timeout)
|
||||
})
|
||||
|
||||
if (response && response.status === 200) {
|
||||
if (/^#EXTM3U/.test(response.data)) {
|
||||
const resolution = parseResolution(response.data)
|
||||
if (resolution) {
|
||||
channel.resolution = resolution
|
||||
}
|
||||
log.print(`\nProcessing '${playlist.url}'...`)
|
||||
await parser
|
||||
.parsePlaylist(playlist.url)
|
||||
.then(formatPlaylist)
|
||||
.then(playlist => {
|
||||
if (file.read(playlist.url) !== playlist.toString()) {
|
||||
log.print('updated')
|
||||
playlist.updated = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results.push(channel)
|
||||
playlist.save()
|
||||
})
|
||||
}
|
||||
|
||||
playlist.channels = results
|
||||
log.print('\n')
|
||||
log.finish()
|
||||
}
|
||||
|
||||
async function formatPlaylist(playlist) {
|
||||
for (const channel of playlist.channels) {
|
||||
const code = file.getBasename(playlist.url)
|
||||
// add missing tvg-name
|
||||
if (!channel.tvg.name && code !== 'unsorted' && channel.name) {
|
||||
channel.tvg.name = channel.name.replace(/\"/gi, '')
|
||||
}
|
||||
// add missing tvg-id
|
||||
if (!channel.tvg.id && code !== 'unsorted' && channel.tvg.name) {
|
||||
const id = utils.name2id(channel.tvg.name)
|
||||
channel.tvg.id = id ? `${id}.${code}` : ''
|
||||
}
|
||||
// add missing country
|
||||
if (!channel.countries.length) {
|
||||
const name = utils.code2name(code)
|
||||
channel.countries = name ? [{ code, name }] : []
|
||||
channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';')
|
||||
}
|
||||
// update group-title
|
||||
channel.group.title = channel.category
|
||||
}
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
function parseResolution(string) {
|
||||
const regex = /RESOLUTION=(\d+)x(\d+)/gm
|
||||
const match = string.matchAll(regex)
|
||||
const arr = Array.from(match).map(m => ({
|
||||
width: parseInt(m[1]),
|
||||
height: parseInt(m[2])
|
||||
}))
|
||||
|
||||
return arr.length
|
||||
? arr.reduce(function (prev, current) {
|
||||
return prev.height > current.height ? prev : current
|
||||
})
|
||||
: undefined
|
||||
}
|
||||
|
||||
async function savePlaylist(playlist) {
|
||||
const original = utils.readFile(playlist.url)
|
||||
const output = playlist.toString()
|
||||
|
||||
if (original === output) {
|
||||
console.info(`No changes have been made.`)
|
||||
return false
|
||||
} else {
|
||||
utils.createFile(playlist.url, output)
|
||||
console.info(`Playlist has been updated.`)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
async function done() {
|
||||
console.info(` `)
|
||||
}
|
||||
|
||||
function finish() {
|
||||
console.timeEnd('Done in')
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
const db = require('./db')
|
||||
const utils = require('./utils')
|
||||
const file = require('./helpers/file')
|
||||
const log = require('./helpers/log')
|
||||
const db = require('./helpers/db')
|
||||
|
||||
const ROOT_DIR = './.gh-pages'
|
||||
|
||||
db.load()
|
||||
|
||||
function main() {
|
||||
async function main() {
|
||||
await loadDatabase()
|
||||
createRootDirectory()
|
||||
createNoJekyllFile()
|
||||
generateIndex()
|
||||
@ -16,51 +16,56 @@ function main() {
|
||||
generateCountries()
|
||||
generateLanguages()
|
||||
generateChannelsJson()
|
||||
finish()
|
||||
showResults()
|
||||
}
|
||||
|
||||
async function loadDatabase() {
|
||||
log.print('Loading database...\n')
|
||||
await db.load()
|
||||
}
|
||||
|
||||
function createRootDirectory() {
|
||||
console.log('Creating .gh-pages folder...')
|
||||
utils.createDir(ROOT_DIR)
|
||||
log.print('Creating .gh-pages folder...\n')
|
||||
file.createDir(ROOT_DIR)
|
||||
}
|
||||
|
||||
function createNoJekyllFile() {
|
||||
console.log('Creating .nojekyll...')
|
||||
utils.createFile(`${ROOT_DIR}/.nojekyll`)
|
||||
log.print('Creating .nojekyll...\n')
|
||||
file.create(`${ROOT_DIR}/.nojekyll`)
|
||||
}
|
||||
|
||||
function generateIndex() {
|
||||
console.log('Generating index.m3u...')
|
||||
log.print('Generating index.m3u...\n')
|
||||
const filename = `${ROOT_DIR}/index.m3u`
|
||||
utils.createFile(filename, '#EXTM3U\n')
|
||||
file.create(filename, '#EXTM3U\n')
|
||||
|
||||
const nsfwFilename = `${ROOT_DIR}/index.nsfw.m3u`
|
||||
utils.createFile(nsfwFilename, '#EXTM3U\n')
|
||||
file.create(nsfwFilename, '#EXTM3U\n')
|
||||
|
||||
const channels = db.channels.sortBy(['name', 'url']).removeDuplicates().get()
|
||||
for (const channel of channels) {
|
||||
if (!channel.isNSFW()) {
|
||||
utils.appendToFile(filename, channel.toString())
|
||||
file.append(filename, channel.toString())
|
||||
}
|
||||
utils.appendToFile(nsfwFilename, channel.toString())
|
||||
file.append(nsfwFilename, channel.toString())
|
||||
}
|
||||
}
|
||||
|
||||
function generateCategoryIndex() {
|
||||
console.log('Generating index.category.m3u...')
|
||||
log.print('Generating index.category.m3u...\n')
|
||||
const filename = `${ROOT_DIR}/index.category.m3u`
|
||||
utils.createFile(filename, '#EXTM3U\n')
|
||||
file.create(filename, '#EXTM3U\n')
|
||||
|
||||
const channels = db.channels.sortBy(['category', 'name', 'url']).removeDuplicates().get()
|
||||
for (const channel of channels) {
|
||||
utils.appendToFile(filename, channel.toString())
|
||||
file.append(filename, channel.toString())
|
||||
}
|
||||
}
|
||||
|
||||
function generateCountryIndex() {
|
||||
console.log('Generating index.country.m3u...')
|
||||
log.print('Generating index.country.m3u...\n')
|
||||
const filename = `${ROOT_DIR}/index.country.m3u`
|
||||
utils.createFile(filename, '#EXTM3U\n')
|
||||
file.create(filename, '#EXTM3U\n')
|
||||
|
||||
for (const country of [{ code: 'undefined' }, ...db.countries.sortBy(['name']).all()]) {
|
||||
const channels = db.channels
|
||||
@ -69,21 +74,21 @@ function generateCountryIndex() {
|
||||
.removeDuplicates()
|
||||
.get()
|
||||
for (const channel of channels) {
|
||||
const category = channel.category
|
||||
const groupTitle = channel.group.title
|
||||
const nsfw = channel.isNSFW()
|
||||
channel.category = country.name || ''
|
||||
channel.group.title = country.name || ''
|
||||
if (!nsfw) {
|
||||
utils.appendToFile(filename, channel.toString())
|
||||
file.append(filename, channel.toString())
|
||||
}
|
||||
channel.category = category
|
||||
channel.group.title = groupTitle
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function generateLanguageIndex() {
|
||||
console.log('Generating index.language.m3u...')
|
||||
log.print('Generating index.language.m3u...\n')
|
||||
const filename = `${ROOT_DIR}/index.language.m3u`
|
||||
utils.createFile(filename, '#EXTM3U\n')
|
||||
file.create(filename, '#EXTM3U\n')
|
||||
|
||||
for (const language of [{ code: 'undefined' }, ...db.languages.sortBy(['name']).all()]) {
|
||||
const channels = db.channels
|
||||
@ -92,25 +97,25 @@ function generateLanguageIndex() {
|
||||
.removeDuplicates()
|
||||
.get()
|
||||
for (const channel of channels) {
|
||||
const category = channel.category
|
||||
const groupTitle = channel.group.title
|
||||
const nsfw = channel.isNSFW()
|
||||
channel.category = language.name || ''
|
||||
channel.group.title = language.name || ''
|
||||
if (!nsfw) {
|
||||
utils.appendToFile(filename, channel.toString())
|
||||
file.append(filename, channel.toString())
|
||||
}
|
||||
channel.category = category
|
||||
channel.group.title = groupTitle
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function generateCategories() {
|
||||
console.log(`Generating /categories...`)
|
||||
log.print(`Generating /categories...\n`)
|
||||
const outputDir = `${ROOT_DIR}/categories`
|
||||
utils.createDir(outputDir)
|
||||
file.createDir(outputDir)
|
||||
|
||||
for (const category of [...db.categories.all(), { id: 'other' }]) {
|
||||
const filename = `${outputDir}/${category.id}.m3u`
|
||||
utils.createFile(filename, '#EXTM3U\n')
|
||||
file.create(filename, '#EXTM3U\n')
|
||||
|
||||
const channels = db.channels
|
||||
.sortBy(['name', 'url'])
|
||||
@ -118,19 +123,19 @@ function generateCategories() {
|
||||
.removeDuplicates()
|
||||
.get()
|
||||
for (const channel of channels) {
|
||||
utils.appendToFile(filename, channel.toString())
|
||||
file.append(filename, channel.toString())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function generateCountries() {
|
||||
console.log(`Generating /countries...`)
|
||||
log.print(`Generating /countries...\n`)
|
||||
const outputDir = `${ROOT_DIR}/countries`
|
||||
utils.createDir(outputDir)
|
||||
file.createDir(outputDir)
|
||||
|
||||
for (const country of [...db.countries.all(), { code: 'undefined' }]) {
|
||||
const filename = `${outputDir}/${country.code}.m3u`
|
||||
utils.createFile(filename, '#EXTM3U\n')
|
||||
file.create(filename, '#EXTM3U\n')
|
||||
|
||||
const channels = db.channels
|
||||
.sortBy(['name', 'url'])
|
||||
@ -139,20 +144,20 @@ function generateCountries() {
|
||||
.get()
|
||||
for (const channel of channels) {
|
||||
if (!channel.isNSFW()) {
|
||||
utils.appendToFile(filename, channel.toString())
|
||||
file.append(filename, channel.toString())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function generateLanguages() {
|
||||
console.log(`Generating /languages...`)
|
||||
log.print(`Generating /languages...\n`)
|
||||
const outputDir = `${ROOT_DIR}/languages`
|
||||
utils.createDir(outputDir)
|
||||
file.createDir(outputDir)
|
||||
|
||||
for (const language of [...db.languages.all(), { code: 'undefined' }]) {
|
||||
const filename = `${outputDir}/${language.code}.m3u`
|
||||
utils.createFile(filename, '#EXTM3U\n')
|
||||
file.create(filename, '#EXTM3U\n')
|
||||
|
||||
const channels = db.channels
|
||||
.sortBy(['name', 'url'])
|
||||
@ -161,25 +166,25 @@ function generateLanguages() {
|
||||
.get()
|
||||
for (const channel of channels) {
|
||||
if (!channel.isNSFW()) {
|
||||
utils.appendToFile(filename, channel.toString())
|
||||
file.append(filename, channel.toString())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function generateChannelsJson() {
|
||||
console.log('Generating channels.json...')
|
||||
log.print('Generating channels.json...\n')
|
||||
const filename = `${ROOT_DIR}/channels.json`
|
||||
const channels = db.channels
|
||||
.sortBy(['name', 'url'])
|
||||
.get()
|
||||
.map(c => c.toObject())
|
||||
utils.createFile(filename, JSON.stringify(channels))
|
||||
file.create(filename, JSON.stringify(channels))
|
||||
}
|
||||
|
||||
function finish() {
|
||||
console.log(
|
||||
`\nTotal: ${db.channels.count()} channels, ${db.countries.count()} countries, ${db.languages.count()} languages, ${db.categories.count()} categories.`
|
||||
function showResults() {
|
||||
log.print(
|
||||
`Total: ${db.channels.count()} channels, ${db.countries.count()} countries, ${db.languages.count()} languages, ${db.categories.count()} categories.\n`
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
145
scripts/helpers/Channel.js
Normal file
145
scripts/helpers/Channel.js
Normal file
@ -0,0 +1,145 @@
|
||||
const categories = require('./categories')
|
||||
const utils = require('./utils')
|
||||
const file = require('./file')
|
||||
|
||||
const sfwCategories = categories.filter(c => !c.nsfw).map(c => c.name)
|
||||
const nsfwCategories = categories.filter(c => c.nsfw).map(c => c.name)
|
||||
|
||||
module.exports = class Channel {
|
||||
constructor(data) {
|
||||
this.raw = data.raw
|
||||
this.tvg = data.tvg
|
||||
this.http = data.http
|
||||
this.url = data.url
|
||||
this.logo = data.tvg.logo
|
||||
this.group = data.group
|
||||
this.name = this.parseName(data.name)
|
||||
this.status = this.parseStatus(data.name)
|
||||
this.resolution = this.parseResolution(data.name)
|
||||
this.category = this.parseCategory(data.group.title)
|
||||
this.countries = this.parseCountries(data.tvg.country)
|
||||
this.languages = this.parseLanguages(data.tvg.language)
|
||||
}
|
||||
|
||||
parseName(title) {
|
||||
return title
|
||||
.trim()
|
||||
.split(' ')
|
||||
.map(s => s.trim())
|
||||
.filter(s => {
|
||||
return !/\[|\]/i.test(s) && !/\((\d+)P\)/i.test(s)
|
||||
})
|
||||
.join(' ')
|
||||
}
|
||||
|
||||
parseStatus(title) {
|
||||
const match = title.match(/\[(.*)\]/i)
|
||||
return match ? match[1] : null
|
||||
}
|
||||
|
||||
parseResolution(title) {
|
||||
const match = title.match(/\((\d+)P\)/i)
|
||||
const height = match ? parseInt(match[1]) : null
|
||||
|
||||
return { width: null, height }
|
||||
}
|
||||
|
||||
parseCategory(string) {
|
||||
const category = categories.find(c => c.id === string.toLowerCase())
|
||||
if (!category) return ''
|
||||
|
||||
return category.name
|
||||
}
|
||||
|
||||
parseCountries(string) {
|
||||
const list = string.split(';')
|
||||
return list
|
||||
.reduce((acc, curr) => {
|
||||
const codes = utils.region2codes(curr)
|
||||
if (codes.length) {
|
||||
for (let code of codes) {
|
||||
if (!acc.includes(code)) {
|
||||
acc.push(code)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
acc.push(curr)
|
||||
}
|
||||
|
||||
return acc
|
||||
}, [])
|
||||
.map(code => {
|
||||
const name = code ? utils.code2name(code) : null
|
||||
if (!name) return null
|
||||
|
||||
return { code: code.toLowerCase(), name }
|
||||
})
|
||||
.filter(c => c)
|
||||
}
|
||||
|
||||
parseLanguages(string) {
|
||||
const list = string.split(';')
|
||||
return list
|
||||
.map(name => {
|
||||
const code = name ? utils.language2code(name) : null
|
||||
if (!code) return null
|
||||
|
||||
return { code, name }
|
||||
})
|
||||
.filter(l => l)
|
||||
}
|
||||
|
||||
isSFW() {
|
||||
return sfwCategories.includes(this.category)
|
||||
}
|
||||
|
||||
isNSFW() {
|
||||
return nsfwCategories.includes(this.category)
|
||||
}
|
||||
|
||||
getInfo() {
|
||||
let info = `-1 tvg-id="${this.tvg.id}" tvg-name="${this.tvg.name}" tvg-country="${this.tvg.country}" tvg-language="${this.tvg.language}" tvg-logo="${this.logo}"`
|
||||
|
||||
info += ` group-title="${this.group.title}",${this.name}`
|
||||
|
||||
if (this.resolution.height) {
|
||||
info += ` (${this.resolution.height}p)`
|
||||
}
|
||||
|
||||
if (this.status) {
|
||||
info += ` [${this.status}]`
|
||||
}
|
||||
|
||||
if (this.http['referrer']) {
|
||||
info += `\n#EXTVLCOPT:http-referrer=${this.http['referrer']}`
|
||||
}
|
||||
|
||||
if (this.http['user-agent']) {
|
||||
info += `\n#EXTVLCOPT:http-user-agent=${this.http['user-agent']}`
|
||||
}
|
||||
|
||||
return info
|
||||
}
|
||||
|
||||
toString(raw = false) {
|
||||
if (raw) return this.raw + '\n'
|
||||
|
||||
return '#EXTINF:' + this.getInfo() + '\n' + this.url + '\n'
|
||||
}
|
||||
|
||||
toObject() {
|
||||
return {
|
||||
name: this.name,
|
||||
logo: this.logo || null,
|
||||
url: this.url,
|
||||
category: this.category || null,
|
||||
languages: this.languages,
|
||||
countries: this.countries,
|
||||
tvg: {
|
||||
id: this.tvg.id || null,
|
||||
name: this.tvg.name || null,
|
||||
url: this.tvg.url || null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
37
scripts/helpers/Playlist.js
Normal file
37
scripts/helpers/Playlist.js
Normal file
@ -0,0 +1,37 @@
|
||||
const Channel = require('./Channel')
|
||||
const file = require('./file')
|
||||
|
||||
module.exports = class Playlist {
|
||||
constructor({ header, items, url, name, country }) {
|
||||
this.url = url
|
||||
this.name = name
|
||||
this.country = country
|
||||
this.header = header
|
||||
this.channels = items.map(item => new Channel(item)).filter(channel => channel.url)
|
||||
this.updated = false
|
||||
}
|
||||
|
||||
toString(options = {}) {
|
||||
const config = { raw: false, ...options }
|
||||
let parts = ['#EXTM3U']
|
||||
for (let key in this.header.attrs) {
|
||||
let value = this.header.attrs[key]
|
||||
if (value) {
|
||||
parts.push(`${key}="${value}"`)
|
||||
}
|
||||
}
|
||||
|
||||
let output = `${parts.join(' ')}\n`
|
||||
for (let channel of this.channels) {
|
||||
output += channel.toString(config.raw)
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
save() {
|
||||
if (this.updated) {
|
||||
file.create(this.url, this.toString())
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -2,14 +2,12 @@ const categories = require('./categories')
|
||||
const parser = require('./parser')
|
||||
const utils = require('./utils')
|
||||
|
||||
const sfwCategories = categories.filter(c => !c.nsfw).map(c => c.name)
|
||||
|
||||
const db = {}
|
||||
|
||||
db.load = function () {
|
||||
db.load = async function () {
|
||||
const items = parser.parseIndex()
|
||||
for (const item of items) {
|
||||
const playlist = parser.parsePlaylist(item.url)
|
||||
const playlist = await parser.parsePlaylist(item.url)
|
||||
db.playlists.add(playlist)
|
||||
for (const channel of playlist.channels) {
|
||||
db.channels.add(channel)
|
||||
@ -107,9 +105,6 @@ db.channels = {
|
||||
all() {
|
||||
return this.list
|
||||
},
|
||||
sfw() {
|
||||
return this.list.filter(i => sfwCategories.includes(i.category))
|
||||
},
|
||||
forCountry(country) {
|
||||
this.filter = {
|
||||
field: 'countries',
|
||||
38
scripts/helpers/file.js
Normal file
38
scripts/helpers/file.js
Normal file
@ -0,0 +1,38 @@
|
||||
const markdownInclude = require('markdown-include')
|
||||
const path = require('path')
|
||||
const fs = require('fs')
|
||||
|
||||
const rootPath = path.resolve(__dirname) + '/../../'
|
||||
const file = {}
|
||||
|
||||
file.getBasename = function (filename) {
|
||||
return path.basename(filename, path.extname(filename))
|
||||
}
|
||||
|
||||
file.getFilename = function (filename) {
|
||||
return path.parse(filename).name
|
||||
}
|
||||
|
||||
file.createDir = function (dir) {
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir)
|
||||
}
|
||||
}
|
||||
|
||||
file.read = function (filename) {
|
||||
return fs.readFileSync(rootPath + filename, { encoding: 'utf8' })
|
||||
}
|
||||
|
||||
file.append = function (filename, data) {
|
||||
fs.appendFileSync(rootPath + filename, data)
|
||||
}
|
||||
|
||||
file.create = function (filename, data = '') {
|
||||
fs.writeFileSync(rootPath + filename, data)
|
||||
}
|
||||
|
||||
file.compileMarkdown = function (filename) {
|
||||
markdownInclude.compileFiles(rootPath + filename)
|
||||
}
|
||||
|
||||
module.exports = file
|
||||
16
scripts/helpers/log.js
Normal file
16
scripts/helpers/log.js
Normal file
@ -0,0 +1,16 @@
|
||||
const log = {}
|
||||
|
||||
log.print = function (string) {
|
||||
process.stdout.write(string)
|
||||
}
|
||||
|
||||
log.start = function () {
|
||||
this.print('Starting...\n')
|
||||
console.time('Done in')
|
||||
}
|
||||
|
||||
log.finish = function () {
|
||||
console.timeEnd('Done in')
|
||||
}
|
||||
|
||||
module.exports = log
|
||||
24
scripts/helpers/parser.js
Normal file
24
scripts/helpers/parser.js
Normal file
@ -0,0 +1,24 @@
|
||||
const playlistParser = require('iptv-playlist-parser')
|
||||
const Playlist = require('./Playlist')
|
||||
const utils = require('./utils')
|
||||
const file = require('./file')
|
||||
|
||||
const parser = {}
|
||||
|
||||
parser.parseIndex = function () {
|
||||
const content = file.read('index.m3u')
|
||||
const result = playlistParser.parse(content)
|
||||
|
||||
return result.items
|
||||
}
|
||||
|
||||
parser.parsePlaylist = async function (url) {
|
||||
const content = file.read(url)
|
||||
const result = playlistParser.parse(content)
|
||||
const name = file.getFilename(url)
|
||||
const country = utils.code2name(name)
|
||||
|
||||
return new Playlist({ header: result.header, items: result.items, url, country, name })
|
||||
}
|
||||
|
||||
module.exports = parser
|
||||
@ -1,21 +1,15 @@
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
const axios = require('axios')
|
||||
const zlib = require('zlib')
|
||||
const urlParser = require('url')
|
||||
const escapeStringRegexp = require('escape-string-regexp')
|
||||
const markdownInclude = require('markdown-include')
|
||||
const iso6393 = require('@freearhey/iso-639-3')
|
||||
const transliteration = require('transliteration')
|
||||
const regions = require('./regions')
|
||||
const iso6393 = require('@freearhey/iso-639-3')
|
||||
const categories = require('./categories')
|
||||
const regions = require('./regions')
|
||||
|
||||
const utils = {}
|
||||
const intlDisplayNames = new Intl.DisplayNames(['en'], {
|
||||
style: 'narrow',
|
||||
type: 'region'
|
||||
})
|
||||
|
||||
const utils = {}
|
||||
|
||||
utils.name2id = function (name) {
|
||||
return transliteration
|
||||
.transliterate(name)
|
||||
@ -66,36 +60,29 @@ utils.sortBy = function (arr, fields) {
|
||||
for (let field of fields) {
|
||||
let propA = a[field] ? a[field].toLowerCase() : ''
|
||||
let propB = b[field] ? b[field].toLowerCase() : ''
|
||||
|
||||
if (propA === 'undefined') {
|
||||
return 1
|
||||
}
|
||||
|
||||
if (propB === 'undefined') {
|
||||
return -1
|
||||
}
|
||||
|
||||
if (propA === 'other') {
|
||||
return 1
|
||||
}
|
||||
|
||||
if (propB === 'other') {
|
||||
return -1
|
||||
}
|
||||
|
||||
if (propA < propB) {
|
||||
return -1
|
||||
}
|
||||
if (propA > propB) {
|
||||
return 1
|
||||
}
|
||||
if (propA === 'undefined') return 1
|
||||
if (propB === 'undefined') return -1
|
||||
if (propA === 'other') return 1
|
||||
if (propB === 'other') return -1
|
||||
if (propA < propB) return -1
|
||||
if (propA > propB) return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
}
|
||||
|
||||
utils.getBasename = function (filename) {
|
||||
return path.basename(filename, path.extname(filename))
|
||||
utils.escapeStringRegexp = function (scring) {
|
||||
return escapeStringRegexp(string)
|
||||
}
|
||||
|
||||
utils.sleep = function (ms) {
|
||||
return function (x) {
|
||||
return new Promise(resolve => setTimeout(() => resolve(x), ms))
|
||||
}
|
||||
}
|
||||
|
||||
utils.removeProtocol = function (string) {
|
||||
return string.replace(/(^\w+:|^)\/\//, '')
|
||||
}
|
||||
|
||||
utils.filterPlaylists = function (arr, include = '', exclude = '') {
|
||||
@ -114,75 +101,4 @@ utils.filterPlaylists = function (arr, include = '', exclude = '') {
|
||||
return arr
|
||||
}
|
||||
|
||||
utils.generateTable = function (data, options) {
|
||||
let output = '<table>\n'
|
||||
|
||||
output += '\t<thead>\n\t\t<tr>'
|
||||
for (let column of options.columns) {
|
||||
output += `<th align="${column.align}">${column.name}</th>`
|
||||
}
|
||||
output += '</tr>\n\t</thead>\n'
|
||||
|
||||
output += '\t<tbody>\n'
|
||||
for (let item of data) {
|
||||
output += '\t\t<tr>'
|
||||
let i = 0
|
||||
for (let prop in item) {
|
||||
const column = options.columns[i]
|
||||
let nowrap = column.nowrap
|
||||
let align = column.align
|
||||
output += `<td align="${align}"${nowrap ? ' nowrap' : ''}>${item[prop]}</td>`
|
||||
i++
|
||||
}
|
||||
output += '</tr>\n'
|
||||
}
|
||||
output += '\t</tbody>\n'
|
||||
|
||||
output += '</table>'
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
utils.createDir = function (dir) {
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir)
|
||||
}
|
||||
}
|
||||
|
||||
utils.readFile = function (filename) {
|
||||
return fs.readFileSync(path.resolve(__dirname) + `/../${filename}`, { encoding: 'utf8' })
|
||||
}
|
||||
|
||||
utils.appendToFile = function (filename, data) {
|
||||
fs.appendFileSync(path.resolve(__dirname) + '/../' + filename, data)
|
||||
}
|
||||
|
||||
utils.compileMarkdown = function (filepath) {
|
||||
return markdownInclude.compileFiles(path.resolve(__dirname, filepath))
|
||||
}
|
||||
|
||||
utils.escapeStringRegexp = function (scring) {
|
||||
return escapeStringRegexp(string)
|
||||
}
|
||||
|
||||
utils.createFile = function (filename, data = '') {
|
||||
fs.writeFileSync(path.resolve(__dirname) + '/../' + filename, data)
|
||||
}
|
||||
|
||||
utils.writeToLog = function (country, msg, url) {
|
||||
var now = new Date()
|
||||
var line = `${country}: ${msg} '${url}'`
|
||||
this.appendToFile('error.log', now.toISOString() + ' ' + line + '\n')
|
||||
}
|
||||
|
||||
utils.sleep = function (ms) {
|
||||
return function (x) {
|
||||
return new Promise(resolve => setTimeout(() => resolve(x), ms))
|
||||
}
|
||||
}
|
||||
|
||||
utils.removeProtocol = function (string) {
|
||||
return string.replace(/(^\w+:|^)\/\//, '')
|
||||
}
|
||||
|
||||
module.exports = utils
|
||||
@ -1,244 +0,0 @@
|
||||
const playlistParser = require('iptv-playlist-parser')
|
||||
const utils = require('./utils')
|
||||
const categories = require('./categories')
|
||||
const path = require('path')
|
||||
|
||||
const sfwCategories = categories.filter(c => !c.nsfw).map(c => c.name)
|
||||
const nsfwCategories = categories.filter(c => c.nsfw).map(c => c.name)
|
||||
|
||||
const parser = {}
|
||||
|
||||
parser.parseIndex = function () {
|
||||
const content = utils.readFile('index.m3u')
|
||||
const result = playlistParser.parse(content)
|
||||
|
||||
return result.items
|
||||
}
|
||||
|
||||
parser.parsePlaylist = function (filename) {
|
||||
const content = utils.readFile(filename)
|
||||
const result = playlistParser.parse(content)
|
||||
const name = path.parse(filename).name
|
||||
const country = utils.code2name(name)
|
||||
|
||||
return new Playlist({ header: result.header, items: result.items, url: filename, country, name })
|
||||
}
|
||||
|
||||
class Playlist {
|
||||
constructor({ header, items, url, name, country }) {
|
||||
this.url = url
|
||||
this.name = name
|
||||
this.country = country
|
||||
this.header = header
|
||||
this.channels = items
|
||||
.map(item => new Channel({ data: item, header, sourceUrl: url }))
|
||||
.filter(channel => channel.url)
|
||||
}
|
||||
|
||||
toString(options = {}) {
|
||||
const config = { raw: false, ...options }
|
||||
let parts = ['#EXTM3U']
|
||||
for (let key in this.header.attrs) {
|
||||
let value = this.header.attrs[key]
|
||||
if (value) {
|
||||
parts.push(`${key}="${value}"`)
|
||||
}
|
||||
}
|
||||
|
||||
let output = `${parts.join(' ')}\n`
|
||||
for (let channel of this.channels) {
|
||||
output += channel.toString(config.raw)
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
}
|
||||
|
||||
class Channel {
|
||||
constructor({ data, header, sourceUrl }) {
|
||||
this.parseData(data)
|
||||
|
||||
this.filename = utils.getBasename(sourceUrl)
|
||||
if (!this.countries.length) {
|
||||
const countryName = utils.code2name(this.filename)
|
||||
this.countries = countryName ? [{ code: this.filename, name: countryName }] : []
|
||||
this.tvg.country = this.countries.map(c => c.code.toUpperCase()).join(';')
|
||||
}
|
||||
}
|
||||
|
||||
parseData(data) {
|
||||
const title = this.parseTitle(data.name)
|
||||
|
||||
this.tvg = data.tvg
|
||||
this.http = data.http
|
||||
this.url = data.url
|
||||
this.logo = data.tvg.logo
|
||||
this.name = title.channelName
|
||||
this.status = title.streamStatus
|
||||
this.resolution = title.streamResolution
|
||||
this.countries = this.parseCountries(data.tvg.country)
|
||||
this.languages = this.parseLanguages(data.tvg.language)
|
||||
this.category = this.parseCategory(data.group.title)
|
||||
this.raw = data.raw
|
||||
}
|
||||
|
||||
parseCountries(string) {
|
||||
let arr = string
|
||||
.split(';')
|
||||
.reduce((acc, curr) => {
|
||||
const codes = utils.region2codes(curr)
|
||||
if (codes.length) {
|
||||
for (let code of codes) {
|
||||
if (!acc.includes(code)) {
|
||||
acc.push(code)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
acc.push(curr)
|
||||
}
|
||||
|
||||
return acc
|
||||
}, [])
|
||||
.filter(code => code && utils.code2name(code))
|
||||
|
||||
return arr.map(code => {
|
||||
return { code: code.toLowerCase(), name: utils.code2name(code) }
|
||||
})
|
||||
}
|
||||
|
||||
parseLanguages(string) {
|
||||
return string
|
||||
.split(';')
|
||||
.map(name => {
|
||||
const code = name ? utils.language2code(name) : null
|
||||
if (!code) return null
|
||||
|
||||
return {
|
||||
code,
|
||||
name
|
||||
}
|
||||
})
|
||||
.filter(l => l)
|
||||
}
|
||||
|
||||
parseCategory(string) {
|
||||
const category = categories.find(c => c.id === string.toLowerCase())
|
||||
|
||||
return category ? category.name : ''
|
||||
}
|
||||
|
||||
parseTitle(title) {
|
||||
const channelName = title
|
||||
.trim()
|
||||
.split(' ')
|
||||
.map(s => s.trim())
|
||||
.filter(s => {
|
||||
return !/\[|\]/i.test(s) && !/\((\d+)P\)/i.test(s)
|
||||
})
|
||||
.join(' ')
|
||||
|
||||
const streamStatusMatch = title.match(/\[(.*)\]/i)
|
||||
const streamStatus = streamStatusMatch ? streamStatusMatch[1] : null
|
||||
|
||||
const streamResolutionMatch = title.match(/\((\d+)P\)/i)
|
||||
const streamResolutionHeight = streamResolutionMatch ? parseInt(streamResolutionMatch[1]) : null
|
||||
const streamResolution = { width: null, height: streamResolutionHeight }
|
||||
|
||||
return { channelName, streamStatus, streamResolution }
|
||||
}
|
||||
|
||||
get tvgCountry() {
|
||||
return this.tvg.country
|
||||
.split(';')
|
||||
.map(code => utils.code2name(code))
|
||||
.join(';')
|
||||
}
|
||||
|
||||
get tvgLanguage() {
|
||||
return this.tvg.language
|
||||
}
|
||||
|
||||
get tvgUrl() {
|
||||
return this.tvg.id && this.tvg.url ? this.tvg.url : ''
|
||||
}
|
||||
|
||||
get tvgId() {
|
||||
if (this.tvg.id) {
|
||||
return this.tvg.id
|
||||
} else if (this.filename !== 'unsorted') {
|
||||
const id = utils.name2id(this.tvgName)
|
||||
|
||||
return id ? `${id}.${this.filename}` : ''
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
get tvgName() {
|
||||
if (this.tvg.name) {
|
||||
return this.tvg.name
|
||||
} else if (this.filename !== 'unsorted') {
|
||||
return this.name.replace(/\"/gi, '')
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
getInfo() {
|
||||
this.tvg.country = this.tvg.country.toUpperCase()
|
||||
|
||||
let info = `-1 tvg-id="${this.tvgId}" tvg-name="${this.tvgName}" tvg-country="${this.tvg.country}" tvg-language="${this.tvg.language}" tvg-logo="${this.logo}"`
|
||||
|
||||
info += ` group-title="${this.category}",${this.name}`
|
||||
|
||||
if (this.resolution.height) {
|
||||
info += ` (${this.resolution.height}p)`
|
||||
}
|
||||
|
||||
if (this.status) {
|
||||
info += ` [${this.status}]`
|
||||
}
|
||||
|
||||
if (this.http['referrer']) {
|
||||
info += `\n#EXTVLCOPT:http-referrer=${this.http['referrer']}`
|
||||
}
|
||||
|
||||
if (this.http['user-agent']) {
|
||||
info += `\n#EXTVLCOPT:http-user-agent=${this.http['user-agent']}`
|
||||
}
|
||||
|
||||
return info
|
||||
}
|
||||
|
||||
toString(raw = false) {
|
||||
if (raw) return this.raw + '\n'
|
||||
|
||||
return '#EXTINF:' + this.getInfo() + '\n' + this.url + '\n'
|
||||
}
|
||||
|
||||
toObject() {
|
||||
return {
|
||||
name: this.name,
|
||||
logo: this.logo || null,
|
||||
url: this.url,
|
||||
category: this.category || null,
|
||||
languages: this.languages,
|
||||
countries: this.countries,
|
||||
tvg: {
|
||||
id: this.tvgId || null,
|
||||
name: this.tvgName || null,
|
||||
url: this.tvgUrl || null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
isSFW() {
|
||||
return sfwCategories.includes(this.category)
|
||||
}
|
||||
|
||||
isNSFW() {
|
||||
return nsfwCategories.includes(this.category)
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = parser
|
||||
@ -1,52 +1,42 @@
|
||||
const parser = require('./parser')
|
||||
const utils = require('./utils')
|
||||
const parser = require('./helpers/parser')
|
||||
const utils = require('./helpers/utils')
|
||||
const log = require('./helpers/log')
|
||||
|
||||
let globalBuffer = []
|
||||
|
||||
async function main() {
|
||||
const playlists = parseIndex()
|
||||
log.start()
|
||||
|
||||
log.print(`Parsing 'index.m3u'...`)
|
||||
const playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
|
||||
for (const playlist of playlists) {
|
||||
await loadPlaylist(playlist.url)
|
||||
log.print(`\nProcessing '${playlist.url}'...`)
|
||||
await parser
|
||||
.parsePlaylist(playlist.url)
|
||||
.then(addToBuffer)
|
||||
.then(removeDuplicates)
|
||||
.then(savePlaylist)
|
||||
.then(done)
|
||||
.then(p => p.save())
|
||||
}
|
||||
|
||||
if (playlists.length) {
|
||||
await loadPlaylist('channels/unsorted.m3u')
|
||||
log.print(`\nProcessing 'channels/unsorted.m3u'...`)
|
||||
await parser
|
||||
.parsePlaylist('channels/unsorted.m3u')
|
||||
.then(removeUnsortedDuplicates)
|
||||
.then(savePlaylist)
|
||||
.then(done)
|
||||
.then(p => p.save())
|
||||
}
|
||||
|
||||
finish()
|
||||
}
|
||||
|
||||
function parseIndex() {
|
||||
console.info(`Parsing 'index.m3u'...`)
|
||||
let playlists = parser.parseIndex()
|
||||
playlists = playlists.filter(i => i.url !== 'channels/unsorted.m3u')
|
||||
console.info(`Found ${playlists.length} playlist(s)\n`)
|
||||
|
||||
return playlists
|
||||
}
|
||||
|
||||
async function loadPlaylist(url) {
|
||||
console.info(`Processing '${url}'...`)
|
||||
return parser.parsePlaylist(url)
|
||||
log.print('\n')
|
||||
log.finish()
|
||||
}
|
||||
|
||||
async function addToBuffer(playlist) {
|
||||
if (playlist.url === 'channels/unsorted.m3u') return playlist
|
||||
globalBuffer = globalBuffer.concat(playlist.channels)
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function removeDuplicates(playlist) {
|
||||
console.info(` Looking for duplicates...`)
|
||||
let buffer = {}
|
||||
const channels = playlist.channels.filter(i => {
|
||||
const url = utils.removeProtocol(i.url)
|
||||
@ -58,13 +48,16 @@ async function removeDuplicates(playlist) {
|
||||
return result
|
||||
})
|
||||
|
||||
playlist.channels = channels
|
||||
if (playlist.channels.length !== channels.length) {
|
||||
log.print('updated')
|
||||
playlist.channels = channels
|
||||
playlist.updated = true
|
||||
}
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function removeUnsortedDuplicates(playlist) {
|
||||
console.info(` Looking for duplicates...`)
|
||||
// locally
|
||||
let buffer = {}
|
||||
let channels = playlist.channels.filter(i => {
|
||||
@ -74,37 +67,18 @@ async function removeUnsortedDuplicates(playlist) {
|
||||
|
||||
return result
|
||||
})
|
||||
|
||||
// globally
|
||||
const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
|
||||
channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
|
||||
if (channels.length === playlist.channels.length) return playlist
|
||||
|
||||
playlist.channels = channels
|
||||
if (channels.length !== playlist.channels.length) {
|
||||
log.print('updated')
|
||||
playlist.channels = channels
|
||||
playlist.updated = true
|
||||
}
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
async function savePlaylist(playlist) {
|
||||
const original = utils.readFile(playlist.url)
|
||||
const output = playlist.toString({ raw: true })
|
||||
|
||||
if (original === output) {
|
||||
console.info(`No changes have been made.`)
|
||||
return false
|
||||
} else {
|
||||
utils.createFile(playlist.url, output)
|
||||
console.info(`Playlist has been updated.`)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
async function done() {
|
||||
console.info(` `)
|
||||
}
|
||||
|
||||
function finish() {
|
||||
console.info('Done.')
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
35
scripts/sort.js
Normal file
35
scripts/sort.js
Normal file
@ -0,0 +1,35 @@
|
||||
const parser = require('./helpers/parser')
|
||||
const utils = require('./helpers/utils')
|
||||
const log = require('./helpers/log')
|
||||
|
||||
async function main() {
|
||||
log.start()
|
||||
|
||||
log.print(`Parsing 'index.m3u'...`)
|
||||
let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u')
|
||||
for (const playlist of playlists) {
|
||||
log.print(`\nProcessing '${playlist.url}'...`)
|
||||
await parser
|
||||
.parsePlaylist(playlist.url)
|
||||
.then(sortChannels)
|
||||
.then(p => p.save())
|
||||
}
|
||||
|
||||
log.print('\n')
|
||||
log.finish()
|
||||
}
|
||||
|
||||
async function sortChannels(playlist) {
|
||||
const channels = [...playlist.channels]
|
||||
utils.sortBy(channels, ['name', 'url'])
|
||||
|
||||
if (JSON.stringify(channels) !== JSON.stringify(playlist.channels)) {
|
||||
log.print('updated')
|
||||
playlist.channels = channels
|
||||
playlist.updated = true
|
||||
}
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
main()
|
||||
@ -1,20 +1,25 @@
|
||||
const utils = require('./utils')
|
||||
const db = require('./db')
|
||||
const parser = require('./parser')
|
||||
const utils = require('./helpers/utils')
|
||||
const file = require('./helpers/file')
|
||||
const log = require('./helpers/log')
|
||||
const db = require('./helpers/db')
|
||||
|
||||
db.load()
|
||||
|
||||
function main() {
|
||||
start()
|
||||
async function main() {
|
||||
log.start()
|
||||
await loadDatabase()
|
||||
generateCategoriesTable()
|
||||
generateCountriesTable()
|
||||
generateLanguagesTable()
|
||||
generateReadme()
|
||||
finish()
|
||||
log.finish()
|
||||
}
|
||||
|
||||
async function loadDatabase() {
|
||||
log.print('Loading database...\n')
|
||||
await db.load()
|
||||
}
|
||||
|
||||
function generateCategoriesTable() {
|
||||
console.log(`Generating categories table...`)
|
||||
log.print('Generating categories table...\n')
|
||||
|
||||
const categories = []
|
||||
for (const category of [...db.categories.all(), { name: 'Other', id: 'other' }]) {
|
||||
@ -25,7 +30,7 @@ function generateCategoriesTable() {
|
||||
})
|
||||
}
|
||||
|
||||
const table = utils.generateTable(categories, {
|
||||
const table = generateTable(categories, {
|
||||
columns: [
|
||||
{ name: 'Category', align: 'left' },
|
||||
{ name: 'Channels', align: 'right' },
|
||||
@ -33,11 +38,11 @@ function generateCategoriesTable() {
|
||||
]
|
||||
})
|
||||
|
||||
utils.createFile('./.readme/_categories.md', table)
|
||||
file.create('./.readme/_categories.md', table)
|
||||
}
|
||||
|
||||
function generateCountriesTable() {
|
||||
console.log(`Generating countries table...`)
|
||||
log.print('Generating countries table...\n')
|
||||
|
||||
const countries = []
|
||||
for (const country of [
|
||||
@ -53,7 +58,7 @@ function generateCountriesTable() {
|
||||
})
|
||||
}
|
||||
|
||||
const table = utils.generateTable(countries, {
|
||||
const table = generateTable(countries, {
|
||||
columns: [
|
||||
{ name: 'Country', align: 'left' },
|
||||
{ name: 'Channels', align: 'right' },
|
||||
@ -61,11 +66,11 @@ function generateCountriesTable() {
|
||||
]
|
||||
})
|
||||
|
||||
utils.createFile('./.readme/_countries.md', table)
|
||||
file.create('./.readme/_countries.md', table)
|
||||
}
|
||||
|
||||
function generateLanguagesTable() {
|
||||
console.log(`Generating languages table...`)
|
||||
log.print('Generating languages table...\n')
|
||||
const languages = []
|
||||
|
||||
for (const language of [
|
||||
@ -79,7 +84,7 @@ function generateLanguagesTable() {
|
||||
})
|
||||
}
|
||||
|
||||
const table = utils.generateTable(languages, {
|
||||
const table = generateTable(languages, {
|
||||
columns: [
|
||||
{ name: 'Language', align: 'left' },
|
||||
{ name: 'Channels', align: 'right' },
|
||||
@ -87,20 +92,41 @@ function generateLanguagesTable() {
|
||||
]
|
||||
})
|
||||
|
||||
utils.createFile('./.readme/_languages.md', table)
|
||||
file.create('./.readme/_languages.md', table)
|
||||
}
|
||||
|
||||
function generateTable(data, options) {
|
||||
let output = '<table>\n'
|
||||
|
||||
output += '\t<thead>\n\t\t<tr>'
|
||||
for (let column of options.columns) {
|
||||
output += `<th align="${column.align}">${column.name}</th>`
|
||||
}
|
||||
output += '</tr>\n\t</thead>\n'
|
||||
|
||||
output += '\t<tbody>\n'
|
||||
for (let item of data) {
|
||||
output += '\t\t<tr>'
|
||||
let i = 0
|
||||
for (let prop in item) {
|
||||
const column = options.columns[i]
|
||||
let nowrap = column.nowrap
|
||||
let align = column.align
|
||||
output += `<td align="${align}"${nowrap ? ' nowrap' : ''}>${item[prop]}</td>`
|
||||
i++
|
||||
}
|
||||
output += '</tr>\n'
|
||||
}
|
||||
output += '\t</tbody>\n'
|
||||
|
||||
output += '</table>'
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
function generateReadme() {
|
||||
console.log(`Generating README.md...`)
|
||||
utils.compileMarkdown('../.readme/config.json')
|
||||
}
|
||||
|
||||
function start() {
|
||||
console.log(`Starting...`)
|
||||
}
|
||||
|
||||
function finish() {
|
||||
console.log(`Done.`)
|
||||
log.print('Generating README.md...\n')
|
||||
file.compileMarkdown('.readme/config.json')
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user