mirror of
https://github.com/iptv-org/iptv.git
synced 2024-09-05 00:01:35 +02:00
177 lines
4.3 KiB
JavaScript
177 lines
4.3 KiB
JavaScript
|
const { db, store, parser, file, logger } = require('../../core')
|
||
|
const statuses = require('../../data/statuses')
|
||
|
const _ = require('lodash')
|
||
|
|
||
|
const items = []
|
||
|
|
||
|
const LOGS_DIR = process.env.LOGS_DIR || 'scripts/logs/cluster/load'
|
||
|
|
||
|
async function main() {
|
||
|
let streams = await loadStreams()
|
||
|
const results = await loadResults()
|
||
|
const origins = await findOrigins(results)
|
||
|
streams = await updateStreams(streams, results, origins)
|
||
|
|
||
|
await updateDatabase(streams)
|
||
|
}
|
||
|
|
||
|
main()
|
||
|
|
||
|
async function loadStreams() {
|
||
|
logger.info('loading streams...')
|
||
|
|
||
|
await db.streams.load()
|
||
|
const streams = await db.streams.find({})
|
||
|
|
||
|
logger.info(`found ${streams.length} streams`)
|
||
|
|
||
|
return streams
|
||
|
}
|
||
|
|
||
|
async function loadResults() {
|
||
|
logger.info('loading results from logs...')
|
||
|
|
||
|
const results = {}
|
||
|
const files = await file.list(`${LOGS_DIR}/cluster_*.log`)
|
||
|
for (const filepath of files) {
|
||
|
const parsed = await parser.parseLogs(filepath)
|
||
|
for (const item of parsed) {
|
||
|
results[item._id] = item
|
||
|
}
|
||
|
}
|
||
|
|
||
|
logger.info(`found ${Object.values(results).length} results`)
|
||
|
|
||
|
return results
|
||
|
}
|
||
|
|
||
|
async function findOrigins(results = {}) {
|
||
|
logger.info('searching for stream origins...')
|
||
|
|
||
|
const origins = {}
|
||
|
for (const { error, requests } of Object.values(results)) {
|
||
|
if (error || !Array.isArray(requests) || !requests.length) continue
|
||
|
|
||
|
let origin = requests.shift()
|
||
|
origin = new URL(origin.url)
|
||
|
for (const request of requests) {
|
||
|
const curr = new URL(request.url)
|
||
|
const key = curr.href.replace(/(^\w+:|^)/, '')
|
||
|
if (!origins[key] && curr.host === origin.host) {
|
||
|
origins[key] = origin.href
|
||
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
|
logger.info(`found ${_.uniq(Object.values(origins)).length} origins`)
|
||
|
|
||
|
return origins
|
||
|
}
|
||
|
|
||
|
async function updateStreams(items = [], results = {}, origins = {}) {
|
||
|
logger.info('updating streams...')
|
||
|
|
||
|
let updated = 0
|
||
|
const output = []
|
||
|
for (const item of items) {
|
||
|
const stream = store.create(item)
|
||
|
const result = results[item._id]
|
||
|
|
||
|
if (result) {
|
||
|
const { error, streams, requests } = result
|
||
|
const resolution = parseResolution(streams)
|
||
|
const origin = findOrigin(requests, origins)
|
||
|
let status = parseStatus(error)
|
||
|
|
||
|
if (status) {
|
||
|
const prevStatus = item.status
|
||
|
if (prevStatus.code === 'not_247')
|
||
|
// not_247 -> * = not_247
|
||
|
status = item.status
|
||
|
else if (prevStatus.code === 'geo_blocked')
|
||
|
// geo_blocked -> * = geo_blocked
|
||
|
status = item.status
|
||
|
else if (status.code === 'geo_blocked')
|
||
|
// * -> geo_blocked = *
|
||
|
status = item.status
|
||
|
else if (prevStatus.code === 'offline' && status.code === 'online')
|
||
|
// offline -> online = not_247
|
||
|
status = statuses['not_247']
|
||
|
|
||
|
stream.set('status', { status })
|
||
|
stream.set('is_broken', { status: stream.get('status') })
|
||
|
}
|
||
|
|
||
|
if (resolution) {
|
||
|
stream.set('resolution', { resolution })
|
||
|
}
|
||
|
|
||
|
if (origin) {
|
||
|
stream.set('url', { url: origin })
|
||
|
}
|
||
|
}
|
||
|
|
||
|
if (stream.changed) {
|
||
|
stream.set('updated', true)
|
||
|
output.push(stream.data())
|
||
|
updated++
|
||
|
}
|
||
|
}
|
||
|
|
||
|
logger.info(`updated ${updated} streams`)
|
||
|
|
||
|
return output
|
||
|
}
|
||
|
|
||
|
async function updateDatabase(streams = []) {
|
||
|
logger.info('updating database...')
|
||
|
|
||
|
for (const stream of streams) {
|
||
|
await db.streams.update({ _id: stream._id }, stream)
|
||
|
}
|
||
|
db.streams.compact()
|
||
|
|
||
|
logger.info('done')
|
||
|
}
|
||
|
|
||
|
function findOrigin(requests = [], origins = {}) {
|
||
|
if (origins && Array.isArray(requests)) {
|
||
|
requests = requests.map(r => r.url.replace(/(^\w+:|^)/, ''))
|
||
|
for (const url of requests) {
|
||
|
if (origins[url]) {
|
||
|
return origins[url]
|
||
|
}
|
||
|
}
|
||
|
}
|
||
|
|
||
|
return null
|
||
|
}
|
||
|
|
||
|
function parseResolution(streams) {
|
||
|
const resolution = streams
|
||
|
.filter(s => s.codec_type === 'video')
|
||
|
.reduce(
|
||
|
(acc, curr) => {
|
||
|
if (curr.height > acc.height) return { width: curr.width, height: curr.height }
|
||
|
return acc
|
||
|
},
|
||
|
{ width: 0, height: 0 }
|
||
|
)
|
||
|
|
||
|
if (resolution.width > 0 && resolution.height > 0) return resolution
|
||
|
return null
|
||
|
}
|
||
|
|
||
|
function parseStatus(error) {
|
||
|
if (error) {
|
||
|
if (error.includes('timed out')) {
|
||
|
return statuses['timeout']
|
||
|
} else if (error.includes('403')) {
|
||
|
return statuses['geo_blocked']
|
||
|
}
|
||
|
return statuses['offline']
|
||
|
}
|
||
|
|
||
|
return statuses['online']
|
||
|
}
|