Browse Source

Fix infinite playlist import

Using an hard videos limit in config
develop
Chocobozzz 1 week ago
parent
commit
4efa5535cc
No known key found for this signature in database GPG Key ID: 583A612D890159BE
8 changed files with 56 additions and 4 deletions
  1. +3
    -0
      config/default.yaml
  2. +3
    -0
      config/production.yaml.example
  3. +1
    -0
      server/initializers/checker-before-init.ts
  4. +3
    -0
      server/initializers/config.ts
  5. +2
    -1
      server/lib/job-queue/handlers/video-channel-import.ts
  6. +1
    -1
      server/lib/sync-channel.ts
  7. +42
    -1
      server/tests/api/videos/channel-import-videos.ts
  8. +1
    -1
      server/tests/api/videos/video-channel-syncs.ts

+ 3
- 0
config/default.yaml View File

@@ -578,6 +578,9 @@ import:
# Number of latest published videos to check and to potentially import when syncing a channel
videos_limit_per_synchronization: 10

# Max number of videos to import when the user asks for full sync
full_sync_videos_limit: 1000

auto_blacklist:
# New videos automatically blacklisted so moderators can review before publishing
videos:


+ 3
- 0
config/production.yaml.example View File

@@ -588,6 +588,9 @@ import:
# Number of latest published videos to check and to potentially import when syncing a channel
videos_limit_per_synchronization: 10

# Max number of videos to import when the user asks for full sync
full_sync_videos_limit: 1000

auto_blacklist:
# New videos automatically blacklisted so moderators can review before publishing
videos:


+ 1
- 0
server/initializers/checker-before-init.ts View File

@@ -35,6 +35,7 @@ function checkMissedConfig () {
'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
'import.video_channel_synchronization.full_sync_videos_limit',
'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days',
'client.videos.miniature.display_author_avatar',
'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth',


+ 3
- 0
server/initializers/config.ts View File

@@ -418,6 +418,9 @@ const CONFIG = {
get CHECK_INTERVAL () { return parseDurationToMs(config.get<string>('import.video_channel_synchronization.check_interval')) },
get VIDEOS_LIMIT_PER_SYNCHRONIZATION () {
return config.get<number>('import.video_channel_synchronization.videos_limit_per_synchronization')
},
get FULL_SYNC_VIDEOS_LIMIT () {
return config.get<number>('import.video_channel_synchronization.full_sync_videos_limit')
}
}
},


+ 2
- 1
server/lib/job-queue/handlers/video-channel-import.ts View File

@@ -37,6 +37,7 @@ export async function processVideoChannelImport (job: Job) {
await synchronizeChannel({
channel: videoChannel,
externalChannelUrl: payload.externalChannelUrl,
channelSync
channelSync,
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT
})
}

+ 1
- 1
server/lib/sync-channel.ts View File

@@ -12,8 +12,8 @@ import { ServerConfigManager } from './server-config-manager'
export async function synchronizeChannel (options: {
channel: MChannelAccountDefault
externalChannelUrl: string
videosCountLimit: number
channelSync?: MChannelSync
videosCountLimit?: number
onlyAfter?: Date
}) {
const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options


+ 42
- 1
server/tests/api/videos/channel-import-videos.ts View File

@@ -109,6 +109,45 @@ describe('Test videos import in a channel', function () {
}
})

it('Should limit max amount of videos synced on full sync', async function () {
this.timeout(240_000)

await server.kill()
await server.run({
import: {
video_channel_synchronization: {
full_sync_videos_limit: 1
}
}
})

const { id } = await server.channels.create({ attributes: { name: 'channel3' } })
const channel3Id = id

const { videoChannelSync } = await server.channelSyncs.create({
attributes: {
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
videoChannelId: channel3Id
}
})
const syncId = videoChannelSync.id

await waitJobs(server)

await server.channels.importVideos({
channelName: 'channel3',
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
videoChannelSyncId: syncId
})

await waitJobs(server)

const { total, data } = await server.videos.listByChannel({ handle: 'channel3' })

expect(total).to.equal(1)
expect(data).to.have.lengthOf(1)
})

after(async function () {
await server?.kill()
})
@@ -116,5 +155,7 @@ describe('Test videos import in a channel', function () {
}

runSuite('yt-dlp')
runSuite('youtube-dl')

// FIXME: With recent changes on youtube, youtube-dl doesn't fetch live replays which means the test suite fails
// runSuite('youtube-dl')
})

+ 1
- 1
server/tests/api/videos/video-channel-syncs.ts View File

@@ -220,7 +220,7 @@ describe('Test channel synchronizations', function () {
expect(total).to.equal(0)
})

// FIXME: youtube-dl doesn't work when speicifying a port after the hostname
// FIXME: youtube-dl/yt-dlp doesn't work when speicifying a port after the hostname
// it('Should import a remote PeerTube channel', async function () {
// this.timeout(240_000)



Loading…
Cancel
Save