Browse Source

Fix broken local actors

Some channels can't federate because they don't have public/private
keys, maybe because the generation failed for various reasons
tags/v3.1.0-rc.1
Chocobozzz 1 year ago
parent
commit
8795d6f254
No known key found for this signature in database GPG Key ID: 583A612D890159BE
11 changed files with 90 additions and 26 deletions
  1. +2
    -0
      client/src/app/+admin/system/jobs/jobs.component.ts
  2. +3
    -3
      client/src/app/+videos/video-list/trending/video-trending-header.component.ts
  3. +1
    -1
      server/controllers/api/users/index.ts
  4. +3
    -4
      server/controllers/api/video-channel.ts
  5. +5
    -2
      server/initializers/constants.ts
  6. +34
    -0
      server/initializers/migrations/0605-actor-missing-keys.ts
  7. +8
    -12
      server/lib/activitypub/actor.ts
  8. +20
    -0
      server/lib/job-queue/handlers/actor-keys.ts
  9. +5
    -0
      server/lib/job-queue/job-queue.ts
  10. +4
    -4
      server/lib/user.ts
  11. +5
    -0
      shared/models/server/job.model.ts

+ 2
- 0
client/src/app/+admin/system/jobs/jobs.component.ts View File

@@ -22,11 +22,13 @@ export class JobsComponent extends RestTable implements OnInit {
jobType: JobTypeClient = 'all'
jobTypes: JobTypeClient[] = [
'all',

'activitypub-follow',
'activitypub-http-broadcast',
'activitypub-http-fetcher',
'activitypub-http-unicast',
'activitypub-refresher',
'actor-keys',
'email',
'video-file-import',
'video-import',


+ 3
- 3
client/src/app/+videos/video-list/trending/video-trending-header.component.ts View File

@@ -39,21 +39,21 @@ export class VideoTrendingHeaderComponent extends VideoListHeaderComponent imple
label: $localize`:A variant of Trending videos based on the number of recent interactions, minus user history:Best`,
iconName: 'award',
value: 'best',
tooltip: $localize`Videos totalizing the most interactions for recent videos, minus user history`,
tooltip: $localize`Videos with the most interactions for recent videos, minus user history`,
hidden: true
},
{
label: $localize`:A variant of Trending videos based on the number of recent interactions:Hot`,
iconName: 'flame',
value: 'hot',
tooltip: $localize`Videos totalizing the most interactions for recent videos`,
tooltip: $localize`Videos with the most interactions for recent videos`,
hidden: true
},
{
label: $localize`:Main variant of Trending videos based on number of recent views:Views`,
iconName: 'trending',
value: 'most-viewed',
tooltip: $localize`Videos totalizing the most views during the last 24 hours`
tooltip: $localize`Videos with the most views during the last 24 hours`
},
{
label: $localize`:A variant of Trending videos based on the number of likes:Likes`,


+ 1
- 1
server/controllers/api/users/index.ts View File

@@ -221,7 +221,7 @@ async function createUser (req: express.Request, res: express.Response) {
id: account.id
}
}
}).end()
})
}

async function registerUser (req: express.Request, res: express.Response) {


+ 3
- 4
server/controllers/api/video-channel.ts View File

@@ -3,6 +3,7 @@ import { Hooks } from '@server/lib/plugins/hooks'
import { getServerActor } from '@server/models/application/application'
import { MChannelAccountDefault } from '@server/types/models'
import { VideoChannelCreate, VideoChannelUpdate } from '../../../shared'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger'
import { resetSequelizeInstance } from '../../helpers/database-utils'
import { buildNSFWFilter, createReqFiles, getCountVideos, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils'
@@ -11,7 +12,6 @@ import { getFormattedObjects } from '../../helpers/utils'
import { CONFIG } from '../../initializers/config'
import { MIMETYPES } from '../../initializers/constants'
import { sequelizeTypescript } from '../../initializers/database'
import { setAsyncActorKeys } from '../../lib/activitypub/actor'
import { sendUpdateActor } from '../../lib/activitypub/send'
import { deleteLocalActorAvatarFile, updateLocalActorAvatarFile } from '../../lib/avatar'
import { JobQueue } from '../../lib/job-queue'
@@ -39,7 +39,6 @@ import { AccountModel } from '../../models/account/account'
import { VideoModel } from '../../models/video/video'
import { VideoChannelModel } from '../../models/video/video-channel'
import { VideoPlaylistModel } from '../../models/video/video-playlist'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'

const auditLogger = auditLoggerFactory('channels')
const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR })
@@ -168,8 +167,8 @@ async function addVideoChannel (req: express.Request, res: express.Response) {
return createLocalVideoChannel(videoChannelInfo, account, t)
})

setAsyncActorKeys(videoChannelCreated.Actor)
.catch(err => logger.error('Cannot set async actor keys for account %s.', videoChannelCreated.Actor.url, { err }))
const payload = { actorId: videoChannelCreated.actorId }
await JobQueue.Instance.createJobWithPromise({ type: 'actor-keys', payload })

auditLogger.create(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelCreated.toFormattedJSON()))
logger.info('Video channel %s created.', videoChannelCreated.Actor.url)


+ 5
- 2
server/initializers/constants.ts View File

@@ -1,8 +1,8 @@
import { randomInt } from '../../shared/core-utils/miscs/miscs'
import { CronRepeatOptions, EveryRepeatOptions } from 'bull'
import { randomBytes } from 'crypto'
import { invert } from 'lodash'
import { join } from 'path'
import { randomInt } from '../../shared/core-utils/miscs/miscs'
import {
AbuseState,
JobType,
@@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'

// ---------------------------------------------------------------------------

const LAST_MIGRATION_VERSION = 600
const LAST_MIGRATION_VERSION = 605

// ---------------------------------------------------------------------------

@@ -141,6 +141,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
'video-transcoding': 1,
'video-import': 1,
'email': 5,
'actor-keys': 3,
'videos-views': 1,
'activitypub-refresher': 1,
'video-redundancy': 1,
@@ -153,6 +154,7 @@ const JOB_CONCURRENCY: { [id in JobType]?: number } = {
'activitypub-follow': 1,
'video-file-import': 1,
'email': 5,
'actor-keys': 1,
'videos-views': 1,
'activitypub-refresher': 1,
'video-redundancy': 1,
@@ -167,6 +169,7 @@ const JOB_TTL: { [id in JobType]: number } = {
'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long
'video-import': 1000 * 3600 * 2, // 2 hours
'email': 60000 * 10, // 10 minutes
'actor-keys': 60000 * 20, // 20 minutes
'videos-views': undefined, // Unlimited
'activitypub-refresher': 60000 * 10, // 10 minutes
'video-redundancy': 1000 * 3600 * 3, // 3 hours


+ 34
- 0
server/initializers/migrations/0605-actor-missing-keys.ts View File

@@ -0,0 +1,34 @@
import * as Sequelize from 'sequelize'
import { createPrivateKey, getPublicKey } from '../../helpers/core-utils'
import { PRIVATE_RSA_KEY_SIZE } from '../constants'

async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
db: any
}): Promise<void> {

{
const query = 'SELECT * FROM "actor" WHERE "serverId" IS NULL AND "publicKey" IS NULL'
const options = { type: Sequelize.QueryTypes.SELECT as Sequelize.QueryTypes.SELECT }
const actors = await utils.sequelize.query<any>(query, options)

for (const actor of actors) {
const { key } = await createPrivateKey(PRIVATE_RSA_KEY_SIZE)
const { publicKey } = await getPublicKey(key)

const queryUpdate = `UPDATE "actor" SET "publicKey" = '${publicKey}', "privateKey" = '${key}' WHERE id = ${actor.id}`
await utils.sequelize.query(queryUpdate)
}
}
}

function down (options) {
throw new Error('Not implemented.')
}

export {
up,
down
}

+ 8
- 12
server/lib/activitypub/actor.ts View File

@@ -39,17 +39,13 @@ import { getServerActor } from '@server/models/application/application'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'

// Set account keys, this could be long so process after the account creation and do not block the client
function setAsyncActorKeys <T extends MActor> (actor: T) {
return createPrivateAndPublicKeys()
.then(({ publicKey, privateKey }) => {
actor.publicKey = publicKey
actor.privateKey = privateKey
return actor.save()
})
.catch(err => {
logger.error('Cannot set public/private keys of actor %d.', actor.url, { err })
return actor
})
async function generateAndSaveActorKeys <T extends MActor> (actor: T) {
const { publicKey, privateKey } = await createPrivateAndPublicKeys()

actor.publicKey = publicKey
actor.privateKey = privateKey

return actor.save()
}

function getOrCreateActorAndServerAndModel (
@@ -346,7 +342,7 @@ async function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannel
export {
getOrCreateActorAndServerAndModel,
buildActorInstance,
setAsyncActorKeys,
generateAndSaveActorKeys,
fetchActorTotalItems,
getAvatarInfoIfExists,
updateActorInstance,


+ 20
- 0
server/lib/job-queue/handlers/actor-keys.ts View File

@@ -0,0 +1,20 @@
import * as Bull from 'bull'
import { generateAndSaveActorKeys } from '@server/lib/activitypub/actor'
import { ActorModel } from '@server/models/activitypub/actor'
import { ActorKeysPayload } from '@shared/models'
import { logger } from '../../../helpers/logger'

async function processActorKeys (job: Bull.Job) {
const payload = job.data as ActorKeysPayload
logger.info('Processing email in job %d.', job.id)

const actor = await ActorModel.load(payload.actorId)

await generateAndSaveActorKeys(actor)
}

// ---------------------------------------------------------------------------

export {
processActorKeys
}

+ 5
- 0
server/lib/job-queue/job-queue.ts View File

@@ -7,6 +7,7 @@ import {
ActivitypubHttpBroadcastPayload,
ActivitypubHttpFetcherPayload,
ActivitypubHttpUnicastPayload,
ActorKeysPayload,
EmailPayload,
JobState,
JobType,
@@ -25,6 +26,7 @@ import { processActivityPubHttpBroadcast } from './handlers/activitypub-http-bro
import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher'
import { processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast'
import { refreshAPObject } from './handlers/activitypub-refresher'
import { processActorKeys } from './handlers/actor-keys'
import { processEmail } from './handlers/email'
import { processVideoFileImport } from './handlers/video-file-import'
import { processVideoImport } from './handlers/video-import'
@@ -44,6 +46,7 @@ type CreateJobArgument =
{ type: 'activitypub-refresher', payload: RefreshPayload } |
{ type: 'videos-views', payload: {} } |
{ type: 'video-live-ending', payload: VideoLiveEndingPayload } |
{ type: 'actor-keys', payload: ActorKeysPayload } |
{ type: 'video-redundancy', payload: VideoRedundancyPayload }

type CreateJobOptions = {
@@ -63,6 +66,7 @@ const handlers: { [id in JobType]: (job: Bull.Job) => Promise<any> } = {
'videos-views': processVideosViews,
'activitypub-refresher': refreshAPObject,
'video-live-ending': processVideoLiveEnding,
'actor-keys': processActorKeys,
'video-redundancy': processVideoRedundancy
}

@@ -78,6 +82,7 @@ const jobTypes: JobType[] = [
'videos-views',
'activitypub-refresher',
'video-redundancy',
'actor-keys',
'video-live-ending'
]



+ 4
- 4
server/lib/user.ts View File

@@ -10,7 +10,7 @@ import { UserNotificationSettingModel } from '../models/account/user-notificatio
import { ActorModel } from '../models/activitypub/actor'
import { MAccountDefault, MActorDefault, MChannelActor } from '../types/models'
import { MUser, MUserDefault, MUserId } from '../types/models/user'
import { buildActorInstance, setAsyncActorKeys } from './activitypub/actor'
import { buildActorInstance, generateAndSaveActorKeys } from './activitypub/actor'
import { getLocalAccountActivityPubUrl } from './activitypub/url'
import { Emailer } from './emailer'
import { LiveManager } from './live-manager'
@@ -55,8 +55,8 @@ async function createUserAccountAndChannelAndPlaylist (parameters: {
})

const [ accountActorWithKeys, channelActorWithKeys ] = await Promise.all([
setAsyncActorKeys(account.Actor),
setAsyncActorKeys(videoChannel.Actor)
generateAndSaveActorKeys(account.Actor),
generateAndSaveActorKeys(videoChannel.Actor)
])

account.Actor = accountActorWithKeys
@@ -101,7 +101,7 @@ async function createApplicationActor (applicationId: number) {
type: 'Application'
})

accountCreated.Actor = await setAsyncActorKeys(accountCreated.Actor)
accountCreated.Actor = await generateAndSaveActorKeys(accountCreated.Actor)

return accountCreated
}


+ 5
- 0
shared/models/server/job.model.ts View File

@@ -17,6 +17,7 @@ export type JobType =
| 'activitypub-refresher'
| 'video-redundancy'
| 'video-live-ending'
| 'actor-keys'

export interface Job {
id: number
@@ -131,3 +132,7 @@ export type VideoTranscodingPayload =
export interface VideoLiveEndingPayload {
videoId: number
}

export interface ActorKeysPayload {
actorId: number
}

Loading…
Cancel
Save