Marown/iptv
1
0
Fork 0
mirror of https://github.com/iptv-org/iptv.git synced 2024-07-01 10:06:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

Update remove-duplicates.js

Browse Source
This commit is contained in:
Aleksandr Statciuk 2021-08-12 18:42:30 +03:00
parent f068bbaa1f
commit b81c8ae9f5
1 changed files with 22 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
scripts/remove-duplicates.js
View File

@ -13,7 +13,7 @@ async function main() {
log.print(`\nProcessing '${playlist.url}'...`)
await parser
.parsePlaylist(playlist.url)
.then(addToBuffer)
.then(addToGlobalBuffer)
.then(removeDuplicates)
.then(p => p.save())
}
@ -22,7 +22,8 @@ async function main() {
log.print(`\nProcessing 'channels/unsorted.m3u'...`)
await parser
.parsePlaylist('channels/unsorted.m3u')
.then(removeUnsortedDuplicates)
.then(removeDuplicates)
.then(removeGlobalDuplicates)
.then(p => p.save())
}
@ -30,22 +31,27 @@ async function main() {
log.finish()
}
async function addToBuffer(playlist) {
globalBuffer = globalBuffer.concat(playlist.channels)
async function addToGlobalBuffer(playlist) {
playlist.channels.forEach(channel => {
const url = utils.removeProtocol(channel.url)
globalBuffer.push(url)
})
return playlist
}
async function removeDuplicates(playlist) {
let buffer = {}
const channels = playlist.channels.filter(i => {
const url = utils.removeProtocol(i.url)
const result = typeof buffer[url] === 'undefined'
if (result) {
buffer[url] = true
}
const buffer = []
const channels = playlist.channels.filter(channel => {
const sameUrl = buffer.find(item => {
return utils.removeProtocol(item.url) === utils.removeProtocol(channel.url)
})
if (sameUrl) return false
const sameHash = buffer.find(item => item.hash === channel.hash)
if (sameHash && channel.status === 'Offline') return false
return result
buffer.push(channel)
return true
})
if (playlist.channels.length !== channels.length) {
@ -57,21 +63,12 @@ async function removeDuplicates(playlist) {
return playlist
}
async function removeUnsortedDuplicates(playlist) {
// locally
let buffer = {}
let channels = playlist.channels.filter(i => {
const url = utils.removeProtocol(i.url)
const result = typeof buffer[url] === 'undefined'
if (result) buffer[url] = true
return result
async function removeGlobalDuplicates(playlist) {
const channels = playlist.channels.filter(channel => {
const url = utils.removeProtocol(channel.url)
return !globalBuffer.includes(url)
})
// globally
const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
if (channels.length !== playlist.channels.length) {
log.print('updated')
playlist.channels = channels