Marown/iptv
1
0
Fork 0
mirror of https://github.com/iptv-org/iptv.git synced 2024-07-03 11:06:00 +02:00
Code Issues Packages Projects Releases Wiki Activity

Update remove-duplicates.js

Browse Source
This commit is contained in:
Aleksandr Statciuk 2021-08-12 18:42:30 +03:00
parent f068bbaa1f
commit b81c8ae9f5
1 changed files with 22 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
scripts/remove-duplicates.js
View File

@ -13,7 +13,7 @@ async function main() {
log.print(`\nProcessing '${playlist.url}'...`) log.print(`\nProcessing '${playlist.url}'...`)
await parser await parser
.parsePlaylist(playlist.url) .parsePlaylist(playlist.url)
.then(addToBuffer) .then(addToGlobalBuffer)
.then(removeDuplicates) .then(removeDuplicates)
.then(p => p.save()) .then(p => p.save())
} }
@ -22,7 +22,8 @@ async function main() {
log.print(`\nProcessing 'channels/unsorted.m3u'...`) log.print(`\nProcessing 'channels/unsorted.m3u'...`)
await parser await parser
.parsePlaylist('channels/unsorted.m3u') .parsePlaylist('channels/unsorted.m3u')
.then(removeUnsortedDuplicates) .then(removeDuplicates)
.then(removeGlobalDuplicates)
.then(p => p.save()) .then(p => p.save())
} }
@ -30,22 +31,27 @@ async function main() {
log.finish() log.finish()
} }
async function addToBuffer(playlist) { async function addToGlobalBuffer(playlist) {
globalBuffer = globalBuffer.concat(playlist.channels) playlist.channels.forEach(channel => {
const url = utils.removeProtocol(channel.url)
globalBuffer.push(url)
})
return playlist return playlist
} }
async function removeDuplicates(playlist) { async function removeDuplicates(playlist) {
let buffer = {} const buffer = []
const channels = playlist.channels.filter(i => { const channels = playlist.channels.filter(channel => {
const url = utils.removeProtocol(i.url) const sameUrl = buffer.find(item => {
const result = typeof buffer[url] === 'undefined' return utils.removeProtocol(item.url) === utils.removeProtocol(channel.url)
if (result) { })
buffer[url] = true if (sameUrl) return false
} const sameHash = buffer.find(item => item.hash === channel.hash)
if (sameHash && channel.status === 'Offline') return false
return result buffer.push(channel)
return true
}) })
if (playlist.channels.length !== channels.length) { if (playlist.channels.length !== channels.length) {
@ -57,21 +63,12 @@ async function removeDuplicates(playlist) {
return playlist return playlist
} }
async function removeUnsortedDuplicates(playlist) { async function removeGlobalDuplicates(playlist) {
// locally const channels = playlist.channels.filter(channel => {
let buffer = {} const url = utils.removeProtocol(channel.url)
let channels = playlist.channels.filter(i => { return !globalBuffer.includes(url)
const url = utils.removeProtocol(i.url)
const result = typeof buffer[url] === 'undefined'
if (result) buffer[url] = true
return result
}) })
// globally
const urls = globalBuffer.map(i => utils.removeProtocol(i.url))
channels = channels.filter(i => !urls.includes(utils.removeProtocol(i.url)))
if (channels.length !== playlist.channels.length) { if (channels.length !== playlist.channels.length) {
log.print('updated') log.print('updated')
playlist.channels = channels playlist.channels = channels