Update scripts

This commit is contained in:
freearhey 2023-09-17 04:08:50 +03:00
parent 89b87ad5c8
commit bb2935878d
16 changed files with 248 additions and 231 deletions

View File

@ -1,16 +1,16 @@
import { API_DIR, DB_DIR } from '../../constants'
import { Logger, Database, Collection, Storage } from '../../core'
import { API_DIR, STREAMS_DIR } from '../../constants'
import { Logger, PlaylistParser, Storage } from '../../core'
import { Stream } from '../../models'
async function main() {
const logger = new Logger()
logger.info(`loading streams...`)
const db = new Database(DB_DIR)
const dbStreams = await db.load('streams.db')
const docs = await dbStreams.find({})
const streams = new Collection(docs as any[])
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage: streamsStorage })
const files = await streamsStorage.list('**/*.m3u')
let streams = await parser.parse(files)
streams = streams
.map(data => new Stream(data))
.orderBy((stream: Stream) => stream.channel)
.map((stream: Stream) => stream.toJSON())
@ -18,8 +18,8 @@ async function main() {
logger.info(`found ${streams.count()} streams`)
logger.info('saving to .api/streams.json...')
const storage = new Storage(API_DIR)
await storage.save('streams.json', streams.toJSON())
const apiStorage = new Storage(API_DIR)
await apiStorage.save('streams.json', streams.toJSON())
}
main()

View File

@ -4,7 +4,6 @@ mkdir -p temp/data
curl -L -o temp/data/blocklist.json https://iptv-org.github.io/api/blocklist.json
curl -L -o temp/data/categories.json https://iptv-org.github.io/api/categories.json
curl -L -o temp/data/channels.json https://iptv-org.github.io/api/channels.json
curl -L -o temp/data/streams.json https://iptv-org.github.io/api/streams.json
curl -L -o temp/data/countries.json https://iptv-org.github.io/api/countries.json
curl -L -o temp/data/languages.json https://iptv-org.github.io/api/languages.json
curl -L -o temp/data/regions.json https://iptv-org.github.io/api/regions.json

View File

@ -1,33 +0,0 @@
import { Storage, Logger, PlaylistParser, Collection, Database } from '../../core'
import { Stream, Playlist } from '../../models'
import { STREAMS_DIR, DB_DIR } from '../../constants'
async function main() {
const logger = new Logger()
logger.info(`looking for streams...`)
const storage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage
})
const files = await storage.list(`**/*.m3u`)
let streams = new Collection()
for (let filepath of files) {
const playlist: Playlist = await parser.parse(filepath)
streams = streams.concat(playlist.streams)
}
logger.info(`found ${streams.count()} streams`)
logger.info('clean up the storage...')
const dbStorage = new Storage(DB_DIR)
await dbStorage.clear('streams.db')
logger.info('saving streams to the database...')
const db = new Database(DB_DIR)
const dbStreams = await db.load('streams.db')
const data = streams.map((stream: Stream) => stream.data()).all()
await dbStreams.insert(data)
}
main()

View File

@ -0,0 +1,67 @@
import { STREAMS_DIR, DATA_DIR } from '../../constants'
import { Storage, Logger, PlaylistParser, Collection } from '../../core'
import { Stream, Playlist, Channel } from '../../models'
import { program } from 'commander'
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
async function main() {
const storage = new Storage(STREAMS_DIR)
const logger = new Logger()
logger.info('loading channels from api...')
const dataStorage = new Storage(DATA_DIR)
const channelsContent = await dataStorage.json('channels.json')
const groupedChannels = new Collection(channelsContent)
.map(data => new Channel(data))
.keyBy((channel: Channel) => channel.id)
logger.info('loading streams...')
const parser = new PlaylistParser({ storage })
const files = program.args.length ? program.args : await storage.list('**/*.m3u')
let streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
logger.info('normalizing links...')
streams = streams.map(stream => {
stream.normalizeURL()
return stream
})
logger.info('removing duplicates...')
streams = streams.uniqBy(stream => stream.url)
logger.info('removing wrong id...')
streams = streams.map((stream: Stream) => {
if (groupedChannels.missing(stream.channel)) {
stream.channel = ''
}
return stream
})
logger.info('sorting links...')
streams = streams.orderBy(
[
(stream: Stream) => stream.name,
(stream: Stream) => parseInt(stream.quality.replace('p', '')),
(stream: Stream) => stream.label,
(stream: Stream) => stream.url
],
['asc', 'desc', 'asc', 'asc']
)
logger.info('saving...')
const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
for (let filepath of groupedStreams.keys()) {
const streams = groupedStreams.get(filepath) || []
if (!streams.length) return
const playlist = new Playlist(streams, { public: false })
await storage.save(filepath, playlist.toString())
}
}
main()

View File

@ -1,6 +1,5 @@
import { File, Storage } from '../../core'
import { File, PlaylistParser, Storage } from '../../core'
import { Stream, Category, Channel, Language, Country, Region, Subdivision } from '../../models'
import { Database } from '../../core/database'
import { Collection } from '../../core/collection'
import { Logger } from '../../core/logger'
import _ from 'lodash'
@ -16,32 +15,29 @@ import {
IndexLanguageGenerator,
IndexRegionGenerator
} from '../../generators'
import { DATA_DIR, DB_DIR, LOGS_DIR } from '../../constants'
import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants'
async function main() {
const logger = new Logger()
const dataStorage = new Storage(DATA_DIR)
const storage = new Storage(DATA_DIR)
const channelsContent = await storage.json('channels.json')
logger.info('loading data from api...')
const channelsContent = await dataStorage.json('channels.json')
const channels = new Collection(channelsContent).map(data => new Channel(data))
const categoriesContent = await storage.json('categories.json')
const categoriesContent = await dataStorage.json('categories.json')
const categories = new Collection(categoriesContent).map(data => new Category(data))
const countriesContent = await storage.json('countries.json')
const countriesContent = await dataStorage.json('countries.json')
const countries = new Collection(countriesContent).map(data => new Country(data))
const languagesContent = await storage.json('languages.json')
const languagesContent = await dataStorage.json('languages.json')
const languages = new Collection(languagesContent).map(data => new Language(data))
const regionsContent = await storage.json('regions.json')
const regionsContent = await dataStorage.json('regions.json')
const regions = new Collection(regionsContent).map(data => new Region(data))
const subdivisionsContent = await storage.json('subdivisions.json')
const subdivisionsContent = await dataStorage.json('subdivisions.json')
const subdivisions = new Collection(subdivisionsContent).map(data => new Subdivision(data))
logger.info('loading streams...')
const streams = await loadStreams({ channels, categories, languages })
logger.info(`found ${streams.count()} streams`)
const generatorsLogger = new Logger({
stream: await new Storage(LOGS_DIR).createStream(`generators.log`)
@ -49,7 +45,6 @@ async function main() {
logger.info('generating categories/...')
await new CategoriesGenerator({ categories, streams, logger: generatorsLogger }).generate()
logger.info('generating countries/...')
await new CountriesGenerator({
countries,
@ -58,10 +53,8 @@ async function main() {
subdivisions,
logger: generatorsLogger
}).generate()
logger.info('generating languages/...')
await new LanguagesGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating regions/...')
await new RegionsGenerator({
streams,
@ -69,16 +62,12 @@ async function main() {
subdivisions,
logger: generatorsLogger
}).generate()
logger.info('generating index.m3u...')
await new IndexGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating index.nsfw.m3u...')
await new IndexNsfwGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating index.category.m3u...')
await new IndexCategoryGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating index.country.m3u...')
await new IndexCountryGenerator({
streams,
@ -87,10 +76,8 @@ async function main() {
subdivisions,
logger: generatorsLogger
}).generate()
logger.info('generating index.language.m3u...')
await new IndexLanguageGenerator({ streams, logger: generatorsLogger }).generate()
logger.info('generating index.region.m3u...')
await new IndexRegionGenerator({ streams, regions, logger: generatorsLogger }).generate()
}
@ -110,11 +97,12 @@ async function loadStreams({
const groupedCategories = categories.keyBy(category => category.id)
const groupedLanguages = languages.keyBy(language => language.code)
const db = new Database(DB_DIR)
const dbStreams = await db.load('streams.db')
const docs = await dbStreams.find({})
const streams = new Collection(docs as any[])
.map((data: any) => new Stream(data))
const storage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage })
const files = await storage.list('**/*.m3u')
let streams = await parser.parse(files)
streams = streams
.orderBy([(stream: Stream) => stream.channel, (stream: Stream) => stream.url], ['asc', 'asc'])
.uniqBy((stream: Stream) => stream.channel || _.uniqueId())
.map((stream: Stream) => {

View File

@ -1,6 +1,6 @@
import { DB_DIR, DATA_DIR, STREAMS_DIR } from '../../constants'
import { Database, Storage, Logger, Collection, Dictionary, IssueLoader } from '../../core'
import { Stream, Playlist, Channel } from '../../models'
import { DATA_DIR, STREAMS_DIR } from '../../constants'
import { Storage, Logger, Collection, Dictionary, IssueLoader, PlaylistParser } from '../../core'
import { Stream, Playlist, Channel, Issue } from '../../models'
let processedIssues = new Collection()
let streams: Collection
@ -10,19 +10,19 @@ async function main() {
const logger = new Logger({ disabled: true })
const loader = new IssueLoader()
logger.info('loading streams...')
const db = new Database(DB_DIR)
const docs = await db.load('streams.db')
const dbStreams = await docs.find({})
streams = new Collection(dbStreams as any[]).map(data => new Stream(data))
const storage = new Storage(DATA_DIR)
const channelsContent = await storage.json('channels.json')
logger.info('loading channels from api...')
const dataStorage = new Storage(DATA_DIR)
const channelsContent = await dataStorage.json('channels.json')
groupedChannels = new Collection(channelsContent)
.map(data => new Channel(data))
.keyBy((channel: Channel) => channel.id)
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage: streamsStorage })
const files = await streamsStorage.list('**/*.m3u')
streams = await parser.parse(files)
logger.info('removing broken streams...')
await removeStreams(loader)
@ -32,25 +32,7 @@ async function main() {
logger.info('add new streams...')
await addStreams(loader)
logger.info('normalizing links...')
streams = streams.map(stream => {
stream.normalizeURL()
return stream
})
logger.info('sorting links...')
streams = streams.orderBy(
[
(stream: Stream) => stream.name,
(stream: Stream) => parseInt(stream.quality.replace('p', '')),
(stream: Stream) => stream.label,
(stream: Stream) => stream.url
],
['asc', 'desc', 'asc', 'asc']
)
logger.info('saving...')
const streamsStorage = new Storage(STREAMS_DIR)
const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
for (let filepath of groupedStreams.keys()) {
const streams = groupedStreams.get(filepath) || []
@ -69,19 +51,22 @@ main()
async function removeStreams(loader: IssueLoader) {
const issues = await loader.load({ labels: ['streams:remove', 'approved'] })
issues.forEach((data: Dictionary) => {
issues.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('stream_url')) return
const removed = streams.remove((_stream: Stream) => _stream.url === data.get('stream_url'))
if (removed.notEmpty()) {
processedIssues.add(data.get('issue_number'))
processedIssues.add(issue.number)
}
})
}
async function editStreams(loader: IssueLoader) {
const issues = await loader.load({ labels: ['streams:edit', 'approved'] })
issues.forEach((data: Dictionary) => {
issues.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('stream_url')) return
let stream = streams.first(
@ -111,13 +96,14 @@ async function editStreams(loader: IssueLoader) {
streams.remove((_stream: Stream) => _stream.channel === stream.channel)
streams.add(stream)
processedIssues.add(data.get('issue_number'))
processedIssues.add(issue.number)
})
}
async function addStreams(loader: IssueLoader) {
const issues = await loader.load({ labels: ['streams:add', 'approved'] })
issues.forEach((data: Dictionary) => {
issues.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('channel_id') || data.missing('stream_url')) return
if (streams.includes((_stream: Stream) => _stream.url === data.get('stream_url'))) return
@ -138,6 +124,6 @@ async function addStreams(loader: IssueLoader) {
})
streams.add(stream)
processedIssues.add(data.get('issue_number'))
processedIssues.add(issue.number)
})
}

View File

@ -5,7 +5,6 @@ import chalk from 'chalk'
import { transliterate } from 'transliteration'
import _ from 'lodash'
import { DATA_DIR, STREAMS_DIR } from '../../constants'
import path from 'path'
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
@ -19,73 +18,70 @@ async function main() {
const logger = new Logger()
logger.info(`loading blocklist...`)
const storage = new Storage(DATA_DIR)
const channelsContent = await storage.json('channels.json')
const dataStorage = new Storage(DATA_DIR)
const channelsContent = await dataStorage.json('channels.json')
const channels = new Collection(channelsContent).map(data => new Channel(data))
const blocklistContent = await storage.json('blocklist.json')
const blocklistContent = await dataStorage.json('blocklist.json')
const blocklist = new Collection(blocklistContent).map(data => new Blocked(data))
logger.info(`found ${blocklist.count()} records`)
let errors = new Collection()
let warnings = new Collection()
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage: streamsStorage })
const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
for (const filepath of files) {
const file = new File(filepath)
if (file.extension() !== 'm3u') continue
const streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
let errors = new Collection()
let warnings = new Collection()
let groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
for (const filepath of groupedStreams.keys()) {
const streams = groupedStreams.get(filepath)
if (!streams) continue
const file = new File(filepath)
const [, countryCode] = file.basename().match(/([a-z]{2})(|_.*)\.m3u/i) || [null, '']
const log = new Collection()
const buffer = new Dictionary()
try {
const relativeFilepath = filepath.replace(path.normalize(STREAMS_DIR), '')
const playlist = await parser.parse(relativeFilepath)
playlist.streams.forEach((stream: Stream) => {
const channelNotInDatabase =
stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel)
if (channelNotInDatabase) {
log.add({
type: 'warning',
line: stream.line,
message: `"${stream.channel}" is not in the database`
})
}
streams.forEach((stream: Stream) => {
const channelNotInDatabase =
stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel)
if (channelNotInDatabase) {
log.add({
type: 'warning',
line: stream.line,
message: `"${stream.channel}" is not in the database`
})
}
const alreadyOnPlaylist = stream.url && buffer.has(stream.url)
if (alreadyOnPlaylist) {
log.add({
type: 'warning',
line: stream.line,
message: `"${stream.url}" is already on the playlist`
})
} else {
buffer.set(stream.url, true)
}
const alreadyOnPlaylist = stream.url && buffer.has(stream.url)
if (alreadyOnPlaylist) {
log.add({
type: 'warning',
line: stream.line,
message: `"${stream.url}" is already on the playlist`
})
} else {
buffer.set(stream.url, true)
}
const channelId = generateChannelId(stream.name, countryCode)
const blocked = blocklist.first(
blocked =>
stream.channel.toLowerCase() === blocked.channel.toLowerCase() ||
channelId.toLowerCase() === blocked.channel.toLowerCase()
)
if (blocked) {
log.add({
type: 'error',
line: stream.line,
message: `"${stream.name}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
})
}
})
} catch (error) {
log.add({
type: 'error',
line: 0,
message: error.message.toLowerCase()
})
}
const channelId = generateChannelId(stream.name, countryCode)
const blocked = blocklist.first(
blocked =>
stream.channel.toLowerCase() === blocked.channel.toLowerCase() ||
channelId.toLowerCase() === blocked.channel.toLowerCase()
)
if (blocked) {
log.add({
type: 'error',
line: stream.line,
message: `"${stream.name}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
})
}
})
if (log.notEmpty()) {
logger.info(`\n${chalk.underline(filepath)}`)

View File

@ -1,36 +1,43 @@
import { DATA_DIR } from '../../constants'
import { Collection, Dictionary, IssueLoader, Storage } from '../../core'
import { Blocked, Channel, Stream } from '../../models'
import { DATA_DIR, STREAMS_DIR } from '../../constants'
import { Collection, Dictionary, IssueLoader, Storage, Logger, PlaylistParser } from '../../core'
import { Blocked, Channel, Issue, Stream } from '../../models'
async function main() {
const logger = new Logger()
const loader = new IssueLoader()
const storage = new Storage(DATA_DIR)
logger.info('loading channels from api...')
const channelsContent = await storage.json('channels.json')
const groupedChannels = new Collection(channelsContent)
.map(data => new Channel(data))
.groupBy((channel: Channel) => channel.id)
const streamsContent = await storage.json('streams.json')
const groupedStreams = new Collection(streamsContent)
.map(data => new Stream(data))
.groupBy((stream: Stream) => stream.url)
logger.info('loading blocklist from api...')
const blocklistContent = await storage.json('blocklist.json')
const groupedBlocklist = new Collection(blocklistContent)
.map(data => new Blocked(data))
.groupBy((blocked: Blocked) => blocked.channel)
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({ storage: streamsStorage })
const files = await streamsStorage.list('**/*.m3u')
const streams = await parser.parse(files)
const groupedStreams = streams.groupBy((stream: Stream) => stream.url)
logger.info('loading issue from github...')
const issues = await loader.load({ labels: ['streams:add'] })
logger.info('creating report...')
const buffer = new Dictionary()
const report = issues.map(data => {
const channelId = data.get('channel_id') || undefined
const streamUrl = data.get('stream_url') || undefined
const report = issues.map((issue: Issue) => {
const channelId = issue.data.get('channel_id') || undefined
const streamUrl = issue.data.get('stream_url') || undefined
const result = new Dictionary({
issueNumber: data.get('issue_number'),
issueNumber: issue.number,
channelId,
status: undefined
})

View File

@ -5,7 +5,6 @@ export const README_DIR = process.env.README_DIR || './.readme'
export const API_DIR = process.env.API_DIR || './.api'
export const DATA_DIR = process.env.DATA_DIR || './temp/data'
export const LOGS_DIR = process.env.LOGS_DIR || './temp/logs'
export const DB_DIR = process.env.DB_DIR || './temp/database'
export const TESTING = process.env.NODE_ENV === 'test' ? true : false
export const OWNER = 'iptv-org'
export const REPO = 'iptv'

View File

@ -1,22 +0,0 @@
import Datastore from '@seald-io/nedb'
import * as path from 'path'
export class Database {
rootDir: string
constructor(rootDir: string) {
this.rootDir = rootDir
}
async load(filepath: string) {
const absFilepath = path.join(this.rootDir, filepath)
return new Datastore({
filename: path.resolve(absFilepath),
autoload: true,
onload: (error: Error): any => {
if (error) console.error(error.message)
}
})
}
}

View File

@ -1,4 +1,3 @@
export * from './database'
export * from './logger'
export * from './playlistParser'
export * from './numberParser'

View File

@ -1,33 +1,31 @@
import { Dictionary } from './'
import { Issue } from '../models'
import _ from 'lodash'
const FIELDS = new Dictionary({
'Channel ID': 'channel_id',
'Channel ID (required)': 'channel_id',
'Broken Link': 'stream_url',
'Stream URL': 'stream_url',
'Stream URL (optional)': 'stream_url',
'Stream URL (required)': 'stream_url',
Label: 'label',
Quality: 'quality',
'Channel Name': 'channel_name',
'HTTP User-Agent': 'user_agent',
'HTTP Referrer': 'http_referrer',
Reason: 'reason',
'What happened to the stream?': 'reason',
'Possible Replacement (optional)': 'possible_replacement',
Notes: 'notes',
'Notes (optional)': 'notes'
})
export class IssueParser {
parse(issue: any): Dictionary {
const data = new Dictionary()
data.set('issue_number', issue.number)
const idDict = new Dictionary({
'Channel ID': 'channel_id',
'Channel ID (required)': 'channel_id',
'Broken Link': 'stream_url',
'Stream URL': 'stream_url',
'Stream URL (optional)': 'stream_url',
'Stream URL (required)': 'stream_url',
Label: 'label',
Quality: 'quality',
'Channel Name': 'channel_name',
'HTTP User-Agent': 'user_agent',
'HTTP Referrer': 'http_referrer',
Reason: 'reason',
'What happened to the stream?': 'reason',
'Possible Replacement (optional)': 'possible_replacement',
Notes: 'notes',
'Notes (optional)': 'notes'
})
parse(issue: any): Issue {
const fields = issue.body.split('###')
if (!fields.length) return data
const data = new Dictionary()
fields.forEach((field: string) => {
let [_label, , _value] = field.split(/\r?\n/)
_label = _label ? _label.trim() : ''
@ -35,7 +33,7 @@ export class IssueParser {
if (!_label || !_value) return data
const id: string = idDict.get(_label)
const id: string = FIELDS.get(_label)
const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
if (!id) return
@ -43,6 +41,6 @@ export class IssueParser {
data.set(id, value)
})
return data
return new Issue({ number: issue.number, data })
}
}

View File

@ -1,6 +1,8 @@
import parser from 'iptv-playlist-parser'
import { Playlist, Stream } from '../models'
import { Stream } from '../models'
import { Collection, Storage } from './'
import path from 'path'
import { STREAMS_DIR } from '../constants'
export class PlaylistParser {
storage: Storage
@ -9,7 +11,19 @@ export class PlaylistParser {
this.storage = storage
}
async parse(filepath: string): Promise<Playlist> {
async parse(files: string[]): Promise<Collection> {
let streams = new Collection()
for (let filepath of files) {
const relativeFilepath = filepath.replace(path.normalize(STREAMS_DIR), '')
const _streams: Collection = await this.parseFile(relativeFilepath)
streams = streams.concat(_streams)
}
return streams
}
async parseFile(filepath: string): Promise<Collection> {
const streams = new Collection()
const content = await this.storage.read(filepath)
@ -32,7 +46,7 @@ export class PlaylistParser {
streams.add(stream)
})
return new Playlist(streams)
return streams
}
}

View File

@ -10,10 +10,12 @@ export class Storage {
this.rootDir = path.normalize(rootDir || './')
}
list(pattern: string): Promise<string[]> {
return glob(pattern, {
async list(pattern: string): Promise<string[]> {
const files = await glob(pattern, {
cwd: this.rootDir
})
return files.sort()
}
async createDir(dir: string): Promise<void> {

View File

@ -1,3 +1,4 @@
export * from './issue'
export * from './playlist'
export * from './blocked'
export * from './stream'

16
scripts/models/issue.ts Normal file
View File

@ -0,0 +1,16 @@
import { Dictionary } from '../core'
type IssueProps = {
number: number
data: Dictionary
}
export class Issue {
number: number
data: Dictionary
constructor({ number, data }: IssueProps) {
this.number = number
this.data = data
}
}