Browse Source

Fix sync import of latest videos

tags/v4.3.0
Chocobozzz 1 week ago
parent
commit
3204f4d17d
No known key found for this signature in database GPG Key ID: 583A612D890159BE
5 changed files with 106 additions and 47 deletions
  1. +2
    -2
      server/helpers/youtube-dl/youtube-dl-info-builder.ts
  2. +3
    -1
      server/lib/job-queue/handlers/video-channel-import.ts
  3. +4
    -1
      server/lib/sync-channel.ts
  4. +1
    -1
      server/lib/video-import.ts
  5. +96
    -42
      server/tests/api/videos/video-channel-syncs.ts

+ 2
- 2
server/helpers/youtube-dl/youtube-dl-info-builder.ts View File

@@ -12,7 +12,7 @@ type YoutubeDLInfo = {
tags?: string[]
thumbnailUrl?: string
ext?: string
originallyPublishedAt?: Date
originallyPublishedAtWithoutTime?: Date
webpageUrl?: string

urls?: string[]
@@ -81,7 +81,7 @@ class YoutubeDLInfoBuilder {
tags: this.getTags(obj.tags),
thumbnailUrl: obj.thumbnail || undefined,
urls: this.buildAvailableUrl(obj),
originallyPublishedAt: this.buildOriginallyPublishedAt(obj),
originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj),
ext: obj.ext,
webpageUrl: obj.webpage_url
}


+ 3
- 1
server/lib/job-queue/handlers/video-channel-import.ts View File

@@ -5,7 +5,7 @@ import { synchronizeChannel } from '@server/lib/sync-channel'
import { VideoChannelModel } from '@server/models/video/video-channel'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync'
import { MChannelSync } from '@server/types/models'
import { VideoChannelImportPayload } from '@shared/models'
import { VideoChannelImportPayload, VideoChannelSyncState } from '@shared/models'

export async function processVideoChannelImport (job: Job) {
const payload = job.data as VideoChannelImportPayload
@@ -42,5 +42,7 @@ export async function processVideoChannelImport (job: Job) {
})
} catch (err) {
logger.error(`Failed to import channel ${videoChannel.name}`, { err })
channelSync.state = VideoChannelSyncState.FAILED
await channelSync.save()
}
}

+ 4
- 1
server/lib/sync-channel.ts View File

@@ -93,7 +93,10 @@ async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Dat

const videoInfo = await youtubeDL.getInfoForDownload()

if (videoInfo.originallyPublishedAt.getTime() < onlyAfter.getTime()) {
const onlyAfterWithoutTime = new Date(onlyAfter)
onlyAfterWithoutTime.setHours(0, 0, 0)

if (videoInfo.originallyPublishedAtWithoutTime.getTime() >= onlyAfterWithoutTime.getTime()) {
return true
}
}


+ 1
- 1
server/lib/video-import.ts View File

@@ -125,7 +125,7 @@ async function buildVideoFromImport ({ channelId, importData, importDataOverride
channelId,
originallyPublishedAt: importDataOverride?.originallyPublishedAt
? new Date(importDataOverride?.originallyPublishedAt)
: importData.originallyPublishedAt
: importData.originallyPublishedAtWithoutTime
}

videoData = await Hooks.wrapObject(


+ 96
- 42
server/tests/api/videos/video-channel-syncs.ts View File

@@ -3,9 +3,9 @@ import { FIXTURE_URLS } from '@server/tests/shared'
import { areHttpImportTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode, VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models'
import {
ChannelSyncsCommand,
createSingleServer,
createMultipleServers,
getServerImportConfig,
killallServers,
PeerTubeServer,
setAccessTokensToServers,
setDefaultAccountAvatar,
@@ -20,8 +20,7 @@ describe('Test channel synchronizations', function () {
function runSuite (mode: 'youtube-dl' | 'yt-dlp') {

describe('Sync using ' + mode, function () {
let server: PeerTubeServer
let command: ChannelSyncsCommand
let servers: PeerTubeServer[]

let startTestDate: Date

@@ -35,7 +34,7 @@ describe('Test channel synchronizations', function () {
}

async function changeDateForSync (channelSyncId: number, newDate: string) {
await server.sql.updateQuery(
await servers[0].sql.updateQuery(
`UPDATE "videoChannelSync" ` +
`SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
`WHERE id=${channelSyncId}`
@@ -43,25 +42,23 @@ describe('Test channel synchronizations', function () {
}

before(async function () {
this.timeout(120_000)
this.timeout(240_000)

startTestDate = new Date()

server = await createSingleServer(1, getServerImportConfig(mode))

await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
await setDefaultChannelAvatar([ server ])
await setDefaultAccountAvatar([ server ])
servers = await createMultipleServers(2, getServerImportConfig(mode))

await server.config.enableChannelSync()
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
await setDefaultChannelAvatar(servers)
await setDefaultAccountAvatar(servers)

command = server.channelSyncs
await servers[0].config.enableChannelSync()

{
userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username)
userInfo.accessToken = await servers[0].users.generateUserAndToken(userInfo.username)

const { videoChannels } = await server.users.getMyInfo({ token: userInfo.accessToken })
const { videoChannels } = await servers[0].users.getMyInfo({ token: userInfo.accessToken })
userInfo.channelId = videoChannels[0].id
}
})
@@ -70,9 +67,9 @@ describe('Test channel synchronizations', function () {
this.timeout(120_000)

{
const { video } = await server.imports.importVideo({
const { video } = await servers[0].imports.importVideo({
attributes: {
channelId: server.store.channel.id,
channelId: servers[0].store.channel.id,
privacy: VideoPrivacy.PUBLIC,
targetUrl: FIXTURE_URLS.youtube
}
@@ -80,16 +77,16 @@ describe('Test channel synchronizations', function () {

expect(video.name).to.equal('small video - youtube')

const { total } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
expect(total).to.equal(1)
}

const { videoChannelSync } = await command.create({
const { videoChannelSync } = await servers[0].channelSyncs.create({
attributes: {
externalChannelUrl: FIXTURE_URLS.youtubeChannel,
videoChannelId: server.store.channel.id
videoChannelId: servers[0].store.channel.id
},
token: server.accessToken,
token: servers[0].accessToken,
expectedStatus: HttpStatusCode.OK_200
})
rootChannelSyncId = videoChannelSync.id
@@ -97,16 +94,19 @@ describe('Test channel synchronizations', function () {
// Ensure any missing video not already fetched will be considered as new
await changeDateForSync(videoChannelSync.id, '1970-01-01')

await server.debug.sendCommand({
await servers[0].debug.sendCommand({
body: {
command: 'process-video-channel-sync-latest'
}
})

{
await waitJobs(server)
await waitJobs(servers)

const { total, data } = await server.videos.listByChannel({ handle: 'root_channel', include: VideoInclude.NOT_PUBLISHED_STATE })
const { total, data } = await servers[0].videos.listByChannel({
handle: 'root_channel',
include: VideoInclude.NOT_PUBLISHED_STATE
})
expect(total).to.equal(2)
expect(data[0].name).to.equal('test')
}
@@ -115,18 +115,18 @@ describe('Test channel synchronizations', function () {
it('Should add another synchronization', async function () {
const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar'

const { videoChannelSync } = await command.create({
const { videoChannelSync } = await servers[0].channelSyncs.create({
attributes: {
externalChannelUrl,
videoChannelId: server.store.channel.id
videoChannelId: servers[0].store.channel.id
},
token: server.accessToken,
token: servers[0].accessToken,
expectedStatus: HttpStatusCode.OK_200
})

expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl)
expect(videoChannelSync.channel).to.include({
id: server.store.channel.id,
id: servers[0].store.channel.id,
name: 'root_channel'
})
expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN)
@@ -134,7 +134,7 @@ describe('Test channel synchronizations', function () {
})

it('Should add a synchronization for another user', async function () {
const { videoChannelSync } = await command.create({
const { videoChannelSync } = await servers[0].channelSyncs.create({
attributes: {
externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
videoChannelId: userInfo.channelId
@@ -145,9 +145,9 @@ describe('Test channel synchronizations', function () {
})

it('Should not import a channel if not asked', async function () {
await waitJobs(server)
await waitJobs(servers)

const { data } = await command.listByAccount({ accountName: userInfo.username })
const { data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })

expect(data[0].state).to.contain({
id: VideoChannelSyncState.WAITING_FIRST_RUN,
@@ -160,15 +160,15 @@ describe('Test channel synchronizations', function () {

await changeDateForSync(userInfo.syncId, '2019-03-01')

await server.debug.sendCommand({
await servers[0].debug.sendCommand({
body: {
command: 'process-video-channel-sync-latest'
}
})

await waitJobs(server)
await waitJobs(servers)

const { data, total } = await server.videos.listByChannel({
const { data, total } = await servers[0].videos.listByChannel({
handle: userInfo.channelName,
include: VideoInclude.NOT_PUBLISHED_STATE
})
@@ -180,7 +180,7 @@ describe('Test channel synchronizations', function () {
it('Should list channel synchronizations', async function () {
// Root
{
const { total, data } = await command.listByAccount({ accountName: 'root' })
const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' })
expect(total).to.equal(2)

expect(data[0]).to.deep.contain({
@@ -193,13 +193,13 @@ describe('Test channel synchronizations', function () {

expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate)

expect(data[0].channel).to.contain({ id: server.store.channel.id })
expect(data[0].channel).to.contain({ id: servers[0].store.channel.id })
expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' })
}

// User
{
const { total, data } = await command.listByAccount({ accountName: userInfo.username })
const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
expect(total).to.equal(1)
expect(data[0]).to.deep.contain({
externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux',
@@ -212,7 +212,7 @@ describe('Test channel synchronizations', function () {
})

it('Should list imports of a channel synchronization', async function () {
const { total, data } = await server.imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId })
const { total, data } = await servers[0].imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId })

expect(total).to.equal(1)
expect(data).to.have.lengthOf(1)
@@ -220,14 +220,68 @@ describe('Test channel synchronizations', function () {
})

it('Should remove user\'s channel synchronizations', async function () {
await command.delete({ channelSyncId: userInfo.syncId })
await servers[0].channelSyncs.delete({ channelSyncId: userInfo.syncId })

const { total } = await command.listByAccount({ accountName: userInfo.username })
const { total } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username })
expect(total).to.equal(0)
})

// FIXME: youtube-dl doesn't work when speicifying a port after the hostname
// it('Should import a remote PeerTube channel', async function () {
// this.timeout(240_000)

// await servers[1].videos.quickUpload({ name: 'remote 1' })
// await waitJobs(servers)

// const { videoChannelSync } = await servers[0].channelSyncs.create({
// attributes: {
// externalChannelUrl: servers[1].url + '/c/root_channel',
// videoChannelId: userInfo.channelId
// },
// token: userInfo.accessToken
// })
// await servers[0].channels.importVideos({
// channelName: userInfo.channelName,
// externalChannelUrl: servers[1].url + '/c/root_channel',
// videoChannelSyncId: videoChannelSync.id,
// token: userInfo.accessToken
// })

// await waitJobs(servers)

// const { data, total } = await servers[0].videos.listByChannel({
// handle: userInfo.channelName,
// include: VideoInclude.NOT_PUBLISHED_STATE
// })

// expect(total).to.equal(2)
// expect(data[0].name).to.equal('remote 1')
// })

// it('Should keep synced a remote PeerTube channel', async function () {
// this.timeout(240_000)

// await servers[1].videos.quickUpload({ name: 'remote 2' })
// await waitJobs(servers)

// await servers[0].debug.sendCommand({
// body: {
// command: 'process-video-channel-sync-latest'
// }
// })

// await waitJobs(servers)

// const { data, total } = await servers[0].videos.listByChannel({
// handle: userInfo.channelName,
// include: VideoInclude.NOT_PUBLISHED_STATE
// })
// expect(total).to.equal(2)
// expect(data[0].name).to.equal('remote 2')
// })

after(async function () {
await server?.kill()
await killallServers(servers)
})
})
}


Loading…
Cancel
Save