iptv/scripts/remove-duplicates.js

82 lines
1.9 KiB
JavaScript
Raw Normal View History

2021-08-01 18:56:02 +02:00
const parser = require('./helpers/parser')
const utils = require('./helpers/utils')
2021-09-08 19:57:13 +02:00
const file = require('./helpers/file')
2021-08-01 18:56:02 +02:00
const log = require('./helpers/log')
2021-05-06 15:25:02 +02:00
let globalBuffer = []
async function main() {
2021-08-01 07:04:03 +02:00
log.start()
2021-05-06 15:25:02 +02:00
2021-09-08 19:57:13 +02:00
let files = await file.list()
if (!files.length) log.print(`No files is selected\n`)
files = files.filter(file => file !== 'channels/unsorted.m3u')
for (const file of files) {
log.print(`\nProcessing '${file}'...`)
2021-08-01 07:04:03 +02:00
await parser
2021-09-08 19:57:13 +02:00
.parsePlaylist(file)
2021-08-12 17:42:30 +02:00
.then(addToGlobalBuffer)
2021-08-01 07:04:03 +02:00
.then(removeDuplicates)
2021-08-01 18:56:02 +02:00
.then(p => p.save())
2021-05-06 15:25:02 +02:00
}
2021-09-08 19:57:13 +02:00
if (files.length) {
2021-08-01 07:04:03 +02:00
log.print(`\nProcessing 'channels/unsorted.m3u'...`)
await parser
.parsePlaylist('channels/unsorted.m3u')
2021-08-12 17:42:30 +02:00
.then(removeDuplicates)
.then(removeGlobalDuplicates)
2021-08-01 18:56:02 +02:00
.then(p => p.save())
2021-05-06 15:25:02 +02:00
}
2021-08-01 18:56:02 +02:00
log.print('\n')
2021-08-01 07:04:03 +02:00
log.finish()
2021-05-06 15:25:02 +02:00
}
2021-08-12 17:42:30 +02:00
async function addToGlobalBuffer(playlist) {
playlist.channels.forEach(channel => {
const url = utils.removeProtocol(channel.url)
globalBuffer.push(url)
})
2021-05-06 15:25:02 +02:00
return playlist
}
async function removeDuplicates(playlist) {
2021-08-12 17:42:30 +02:00
const buffer = []
const channels = playlist.channels.filter(channel => {
const sameUrl = buffer.find(item => {
return utils.removeProtocol(item.url) === utils.removeProtocol(channel.url)
})
if (sameUrl) return false
buffer.push(channel)
return true
2021-05-06 15:25:02 +02:00
})
2021-08-01 07:04:03 +02:00
if (playlist.channels.length !== channels.length) {
log.print('updated')
playlist.channels = channels
2021-08-02 18:07:10 +02:00
playlist.updated = true
2021-08-01 07:04:03 +02:00
}
2021-05-06 15:25:02 +02:00
return playlist
}
2021-08-12 17:42:30 +02:00
async function removeGlobalDuplicates(playlist) {
const channels = playlist.channels.filter(channel => {
const url = utils.removeProtocol(channel.url)
return !globalBuffer.includes(url)
2021-05-06 15:25:02 +02:00
})
2021-08-01 07:04:03 +02:00
if (channels.length !== playlist.channels.length) {
log.print('updated')
playlist.channels = channels
2021-08-02 18:07:10 +02:00
playlist.updated = true
2021-05-06 15:25:02 +02:00
}
2021-08-01 07:04:03 +02:00
return playlist
2021-05-06 15:25:02 +02:00
}
main()