Browse Source

Avoir some circular dependencies

tags/v2.2.0-rc.1
Chocobozzz Chocobozzz 2 years ago
parent
commit
8dc8a34ee8
69 changed files with 278 additions and 263 deletions
  1. +1
    -1
      scripts/create-transcoding-job.ts
  2. +3
    -3
      server/controllers/activitypub/client.ts
  3. +2
    -1
      server/controllers/api/accounts.ts
  4. +4
    -2
      server/controllers/api/search.ts
  5. +2
    -1
      server/controllers/api/server/follows.ts
  6. +2
    -1
      server/controllers/api/server/server-blocklist.ts
  7. +3
    -2
      server/controllers/api/video-channel.ts
  8. +2
    -1
      server/controllers/api/video-playlist.ts
  9. +2
    -1
      server/controllers/api/videos/abuse.ts
  10. +1
    -1
      server/controllers/api/videos/blacklist.ts
  11. +1
    -1
      server/controllers/api/videos/captions.ts
  12. +1
    -1
      server/controllers/api/videos/import.ts
  13. +6
    -8
      server/controllers/api/videos/index.ts
  14. +1
    -1
      server/controllers/api/videos/ownership.ts
  15. +1
    -1
      server/controllers/api/videos/rate.ts
  16. +7
    -1
      server/helpers/peertube-crypto.ts
  17. +0
    -13
      server/helpers/utils.ts
  18. +39
    -3
      server/helpers/video.ts
  19. +1
    -1
      server/helpers/webtorrent.ts
  20. +1
    -2
      server/initializers/checker-after-init.ts
  21. +1
    -1
      server/lib/activitypub/actor.ts
  22. +1
    -1
      server/lib/activitypub/follow.ts
  23. +0
    -9
      server/lib/activitypub/index.ts
  24. +1
    -1
      server/lib/activitypub/process/process-follow.ts
  25. +1
    -1
      server/lib/activitypub/send/send-create.ts
  26. +1
    -1
      server/lib/activitypub/send/send-delete.ts
  27. +1
    -1
      server/lib/activitypub/send/send-update.ts
  28. +1
    -1
      server/lib/activitypub/send/utils.ts
  29. +1
    -1
      server/lib/activitypub/share.ts
  30. +1
    -2
      server/lib/activitypub/videos.ts
  31. +1
    -1
      server/lib/avatar.ts
  32. +2
    -12
      server/lib/emailer.ts
  33. +1
    -8
      server/lib/job-queue/handlers/activitypub-follow.ts
  34. +1
    -8
      server/lib/job-queue/handlers/activitypub-http-broadcast.ts
  35. +3
    -10
      server/lib/job-queue/handlers/activitypub-http-fetcher.ts
  36. +1
    -8
      server/lib/job-queue/handlers/activitypub-http-unicast.ts
  37. +4
    -6
      server/lib/job-queue/handlers/activitypub-refresher.ts
  38. +2
    -3
      server/lib/job-queue/handlers/email.ts
  39. +2
    -9
      server/lib/job-queue/handlers/utils/activitypub-http-utils.ts
  40. +1
    -5
      server/lib/job-queue/handlers/video-file-import.ts
  41. +3
    -20
      server/lib/job-queue/handlers/video-import.ts
  42. +1
    -4
      server/lib/job-queue/handlers/video-redundancy.ts
  43. +7
    -35
      server/lib/job-queue/handlers/video-transcoding.ts
  44. +1
    -1
      server/lib/job-queue/handlers/video-views.ts
  45. +17
    -11
      server/lib/job-queue/job-queue.ts
  46. +1
    -1
      server/lib/notifier.ts
  47. +1
    -1
      server/lib/redundancy.ts
  48. +1
    -1
      server/lib/schedulers/auto-follow-index-instances.ts
  49. +1
    -1
      server/lib/schedulers/update-videos-scheduler.ts
  50. +2
    -2
      server/lib/schedulers/videos-redundancy-scheduler.ts
  51. +1
    -1
      server/lib/video-comment.ts
  52. +1
    -1
      server/lib/video-paths.ts
  53. +1
    -1
      server/lib/video-playlist.ts
  54. +0
    -36
      server/lib/videos.ts
  55. +1
    -1
      server/middlewares/activitypub.ts
  56. +1
    -1
      server/middlewares/validators/activitypub/activity.ts
  57. +1
    -1
      server/middlewares/validators/blocklist.ts
  58. +1
    -1
      server/middlewares/validators/follows.ts
  59. +1
    -1
      server/middlewares/validators/videos/videos.ts
  60. +1
    -1
      server/models/activitypub/actor-follow.ts
  61. +11
    -0
      server/models/application/application.ts
  62. +1
    -1
      server/models/redundancy/video-redundancy.ts
  63. +1
    -1
      server/models/video/video-comment.ts
  64. +2
    -2
      server/models/video/video-format-utils.ts
  65. +1
    -1
      server/models/video/video.ts
  66. +2
    -1
      server/tests/api/activitypub/security.ts
  67. +8
    -0
      shared/models/server/emailer.model.ts
  68. +1
    -0
      shared/models/server/index.ts
  69. +100
    -0
      shared/models/server/job.model.ts

+ 1
- 1
scripts/create-transcoding-job.ts View File

@@ -5,8 +5,8 @@ import * as program from 'commander'
import { VideoModel } from '../server/models/video/video'
import { initDatabaseModels } from '../server/initializers'
import { JobQueue } from '../server/lib/job-queue'
import { VideoTranscodingPayload } from '../server/lib/job-queue/handlers/video-transcoding'
import { computeResolutionsToTranscode } from '@server/helpers/ffmpeg-utils'
import { VideoTranscodingPayload } from '@shared/models'

program
.option('-v, --video [videoUUID]', 'Video UUID')


+ 3
- 3
server/controllers/activitypub/client.ts View File

@@ -24,20 +24,20 @@ import { cacheRoute } from '../../middlewares/cache'
import { activityPubResponse } from './utils'
import { AccountVideoRateModel } from '../../models/account/account-video-rate'
import {
getRateUrl,
getVideoCommentsActivityPubUrl,
getVideoDislikesActivityPubUrl,
getVideoLikesActivityPubUrl,
getVideoSharesActivityPubUrl
} from '../../lib/activitypub'
} from '../../lib/activitypub/url'
import { VideoCaptionModel } from '../../models/video/video-caption'
import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy'
import { getServerActor } from '../../helpers/utils'
import { buildDislikeActivity } from '../../lib/activitypub/send/send-dislike'
import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists'
import { VideoPlaylistModel } from '../../models/video/video-playlist'
import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
import { MAccountId, MActorId, MVideoAPWithoutCaption, MVideoId } from '@server/typings/models'
import { getServerActor } from '@server/models/application/application'
import { getRateUrl } from '@server/lib/activitypub/video-rates'

const activityPubClientRouter = express.Router()
activityPubClientRouter.use(cors())


+ 2
- 1
server/controllers/api/accounts.ts View File

@@ -1,5 +1,5 @@
import * as express from 'express'
import { getFormattedObjects, getServerActor } from '../../helpers/utils'
import { getFormattedObjects} from '../../helpers/utils'
import {
asyncMiddleware,
authenticate,
@@ -28,6 +28,7 @@ import { VideoChannelModel } from '../../models/video/video-channel'
import { JobQueue } from '../../lib/job-queue'
import { VideoPlaylistModel } from '../../models/video/video-playlist'
import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists'
import { getServerActor } from '@server/models/application/application'

const accountsRouter = express.Router()



+ 4
- 2
server/controllers/api/search.ts View File

@@ -1,6 +1,6 @@
import * as express from 'express'
import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils'
import { getFormattedObjects, getServerActor } from '../../helpers/utils'
import { getFormattedObjects } from '../../helpers/utils'
import { VideoModel } from '../../models/video/video'
import {
asyncMiddleware,
@@ -15,11 +15,13 @@ import {
videosSearchValidator
} from '../../middlewares'
import { VideoChannelsSearchQuery, VideosSearchQuery } from '../../../shared/models/search'
import { getOrCreateActorAndServerAndModel, getOrCreateVideoAndAccountAndChannel } from '../../lib/activitypub'
import { getOrCreateActorAndServerAndModel } from '../../lib/activitypub/actor'
import { logger } from '../../helpers/logger'
import { VideoChannelModel } from '../../models/video/video-channel'
import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
import { MChannelAccountDefault, MVideoAccountLightBlacklistAllFiles } from '../../typings/models'
import { getServerActor } from '@server/models/application/application'
import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos'

const searchRouter = express.Router()



+ 2
- 1
server/controllers/api/server/follows.ts View File

@@ -1,7 +1,7 @@
import * as express from 'express'
import { UserRight } from '../../../../shared/models/users'
import { logger } from '../../../helpers/logger'
import { getFormattedObjects, getServerActor } from '../../../helpers/utils'
import { getFormattedObjects} from '../../../helpers/utils'
import { SERVER_ACTOR_NAME } from '../../../initializers/constants'
import { sendAccept, sendReject, sendUndoFollow } from '../../../lib/activitypub/send'
import {
@@ -27,6 +27,7 @@ import { JobQueue } from '../../../lib/job-queue'
import { removeRedundanciesOfServer } from '../../../lib/redundancy'
import { sequelizeTypescript } from '../../../initializers/database'
import { autoFollowBackIfNeeded } from '../../../lib/activitypub/follow'
import { getServerActor } from '@server/models/application/application'

const serverFollowsRouter = express.Router()
serverFollowsRouter.get('/following',


+ 2
- 1
server/controllers/api/server/server-blocklist.ts View File

@@ -1,6 +1,6 @@
import * as express from 'express'
import 'multer'
import { getFormattedObjects, getServerActor } from '../../../helpers/utils'
import { getFormattedObjects} from '../../../helpers/utils'
import {
asyncMiddleware,
asyncRetryTransactionMiddleware,
@@ -22,6 +22,7 @@ import { AccountBlocklistModel } from '../../../models/account/account-blocklist
import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../../../lib/blocklist'
import { ServerBlocklistModel } from '../../../models/server/server-blocklist'
import { UserRight } from '../../../../shared/models/users'
import { getServerActor } from '@server/models/application/application'

const serverBlocklistRouter = express.Router()



+ 3
- 2
server/controllers/api/video-channel.ts View File

@@ -1,5 +1,5 @@
import * as express from 'express'
import { getFormattedObjects, getServerActor } from '../../helpers/utils'
import { getFormattedObjects} from '../../helpers/utils'
import {
asyncMiddleware,
asyncRetryTransactionMiddleware,
@@ -21,7 +21,7 @@ import { sendUpdateActor } from '../../lib/activitypub/send'
import { VideoChannelCreate, VideoChannelUpdate } from '../../../shared'
import { createLocalVideoChannel, federateAllVideosOfChannel } from '../../lib/video-channel'
import { buildNSFWFilter, createReqFiles, getCountVideos, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils'
import { setAsyncActorKeys } from '../../lib/activitypub'
import { setAsyncActorKeys } from '../../lib/activitypub/actor'
import { AccountModel } from '../../models/account/account'
import { MIMETYPES } from '../../initializers/constants'
import { logger } from '../../helpers/logger'
@@ -36,6 +36,7 @@ import { commonVideoPlaylistFiltersValidator } from '../../middlewares/validator
import { CONFIG } from '../../initializers/config'
import { sequelizeTypescript } from '../../initializers/database'
import { MChannelAccountDefault } from '@server/typings/models'
import { getServerActor } from '@server/models/application/application'

const auditLogger = auditLoggerFactory('channels')
const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR })


+ 2
- 1
server/controllers/api/video-playlist.ts View File

@@ -1,5 +1,5 @@
import * as express from 'express'
import { getFormattedObjects, getServerActor } from '../../helpers/utils'
import { getFormattedObjects} from '../../helpers/utils'
import {
asyncMiddleware,
asyncRetryTransactionMiddleware,
@@ -41,6 +41,7 @@ import { CONFIG } from '../../initializers/config'
import { sequelizeTypescript } from '../../initializers/database'
import { createPlaylistMiniatureFromExisting } from '../../lib/thumbnail'
import { MVideoPlaylistFull, MVideoPlaylistThumbnail, MVideoThumbnail } from '@server/typings/models'
import { getServerActor } from '@server/models/application/application'

const reqThumbnailFile = createReqFiles([ 'thumbnailfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { thumbnailfile: CONFIG.STORAGE.TMP_DIR })



+ 2
- 1
server/controllers/api/videos/abuse.ts View File

@@ -1,7 +1,7 @@
import * as express from 'express'
import { UserRight, VideoAbuseCreate, VideoAbuseState } from '../../../../shared'
import { logger } from '../../../helpers/logger'
import { getFormattedObjects, getServerActor } from '../../../helpers/utils'
import { getFormattedObjects } from '../../../helpers/utils'
import { sequelizeTypescript } from '../../../initializers'
import {
asyncMiddleware,
@@ -22,6 +22,7 @@ import { auditLoggerFactory, VideoAbuseAuditView } from '../../../helpers/audit-
import { Notifier } from '../../../lib/notifier'
import { sendVideoAbuse } from '../../../lib/activitypub/send/send-flag'
import { MVideoAbuseAccountVideo } from '../../../typings/models/video'
import { getServerActor } from '@server/models/application/application'

const auditLogger = auditLoggerFactory('abuse')
const abuseVideoRouter = express.Router()


+ 1
- 1
server/controllers/api/videos/blacklist.ts View File

@@ -19,7 +19,7 @@ import { VideoBlacklistModel } from '../../../models/video/video-blacklist'
import { sequelizeTypescript } from '../../../initializers'
import { Notifier } from '../../../lib/notifier'
import { sendDeleteVideo } from '../../../lib/activitypub/send'
import { federateVideoIfNeeded } from '../../../lib/activitypub'
import { federateVideoIfNeeded } from '../../../lib/activitypub/videos'
import { MVideoBlacklistVideo } from '@server/typings/models'

const blacklistRouter = express.Router()


+ 1
- 1
server/controllers/api/videos/captions.ts View File

@@ -6,7 +6,7 @@ import { MIMETYPES } from '../../../initializers/constants'
import { getFormattedObjects } from '../../../helpers/utils'
import { VideoCaptionModel } from '../../../models/video/video-caption'
import { logger } from '../../../helpers/logger'
import { federateVideoIfNeeded } from '../../../lib/activitypub'
import { federateVideoIfNeeded } from '../../../lib/activitypub/videos'
import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
import { CONFIG } from '../../../initializers/config'
import { sequelizeTypescript } from '../../../initializers/database'


+ 1
- 1
server/controllers/api/videos/import.ts View File

@@ -10,7 +10,7 @@ import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '.
import { VideoModel } from '../../../models/video/video'
import { VideoCaptionModel } from '../../../models/video/video-caption'
import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils'
import { getVideoActivityPubUrl } from '../../../lib/activitypub'
import { getVideoActivityPubUrl } from '../../../lib/activitypub/url'
import { TagModel } from '../../../models/video/tag'
import { VideoImportModel } from '../../../models/video/video-import'
import { JobQueue } from '../../../lib/job-queue/job-queue'


+ 6
- 8
server/controllers/api/videos/index.ts View File

@@ -4,7 +4,7 @@ import { VideoCreate, VideoPrivacy, VideoState, VideoUpdate } from '../../../../
import { getMetadataFromFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
import { logger } from '../../../helpers/logger'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
import { getFormattedObjects, getServerActor } from '../../../helpers/utils'
import { getFormattedObjects } from '../../../helpers/utils'
import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
import {
DEFAULT_AUDIO_RESOLUTION,
@@ -14,12 +14,7 @@ import {
VIDEO_LICENCES,
VIDEO_PRIVACIES
} from '../../../initializers/constants'
import {
changeVideoChannelShare,
federateVideoIfNeeded,
fetchRemoteVideoDescription,
getVideoActivityPubUrl
} from '../../../lib/activitypub'
import { federateVideoIfNeeded, fetchRemoteVideoDescription } from '../../../lib/activitypub/videos'
import { JobQueue } from '../../../lib/job-queue'
import { Redis } from '../../../lib/redis'
import {
@@ -67,7 +62,10 @@ import { MVideoDetails, MVideoFullLight } from '@server/typings/models'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { getVideoFilePath } from '@server/lib/video-paths'
import toInt from 'validator/lib/toInt'
import { addOptimizeOrMergeAudioJob } from '@server/lib/videos'
import { addOptimizeOrMergeAudioJob } from '@server/helpers/video'
import { getServerActor } from '@server/models/application/application'
import { changeVideoChannelShare } from '@server/lib/activitypub/share'
import { getVideoActivityPubUrl } from '@server/lib/activitypub/url'

const auditLogger = auditLoggerFactory('videos')
const videosRouter = express.Router()


+ 1
- 1
server/controllers/api/videos/ownership.ts View File

@@ -15,7 +15,7 @@ import { VideoChangeOwnershipModel } from '../../../models/video/video-change-ow
import { VideoChangeOwnershipStatus, VideoState } from '../../../../shared/models/videos'
import { VideoChannelModel } from '../../../models/video/video-channel'
import { getFormattedObjects } from '../../../helpers/utils'
import { changeVideoChannelShare } from '../../../lib/activitypub'
import { changeVideoChannelShare } from '../../../lib/activitypub/share'
import { sendUpdateVideo } from '../../../lib/activitypub/send'
import { VideoModel } from '../../../models/video/video'
import { MVideoFullLight } from '@server/typings/models'


+ 1
- 1
server/controllers/api/videos/rate.ts View File

@@ -2,7 +2,7 @@ import * as express from 'express'
import { UserVideoRateUpdate } from '../../../../shared'
import { logger } from '../../../helpers/logger'
import { VIDEO_RATE_TYPES } from '../../../initializers/constants'
import { getRateUrl, sendVideoRateChange } from '../../../lib/activitypub'
import { getRateUrl, sendVideoRateChange } from '../../../lib/activitypub/video-rates'
import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoUpdateRateValidator } from '../../../middlewares'
import { AccountModel } from '../../../models/account/account'
import { AccountVideoRateModel } from '../../../models/account/account-video-rate'


+ 7
- 1
server/helpers/peertube-crypto.ts View File

@@ -5,7 +5,6 @@ import { jsonld } from './custom-jsonld-signature'
import { logger } from './logger'
import { cloneDeep } from 'lodash'
import { createSign, createVerify } from 'crypto'
import { buildDigest } from '../lib/job-queue/handlers/utils/activitypub-http-utils'
import * as bcrypt from 'bcrypt'
import { MActor } from '../typings/models'

@@ -104,12 +103,19 @@ async function signJsonLDObject (byActor: MActor, data: any) {
return Object.assign(data, { signature })
}

function buildDigest (body: any) {
const rawBody = typeof body === 'string' ? body : JSON.stringify(body)

return 'SHA-256=' + sha256(rawBody, 'base64')
}

// ---------------------------------------------------------------------------

export {
isHTTPSignatureDigestValid,
parseHTTPSignature,
isHTTPSignatureVerified,
buildDigest,
isJsonLDSignatureVerified,
comparePassword,
createPrivateAndPublicKeys,


+ 0
- 13
server/helpers/utils.ts View File

@@ -1,11 +1,9 @@
import { ResultList } from '../../shared'
import { ApplicationModel } from '../models/application/application'
import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils'
import { logger } from './logger'
import { join } from 'path'
import { Instance as ParseTorrent } from 'parse-torrent'
import { remove } from 'fs-extra'
import * as memoizee from 'memoizee'
import { CONFIG } from '../initializers/config'
import { isVideoFileExtnameValid } from './custom-validators/videos'

@@ -33,16 +31,6 @@ function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects:
} as ResultList<V>
}

const getServerActor = memoizee(async function () {
const application = await ApplicationModel.load()
if (!application) throw Error('Could not load Application from database.')

const actor = application.Account.Actor
actor.Account = application.Account

return actor
}, { promise: true })

function generateVideoImportTmpPath (target: string | ParseTorrent, extensionArg?: string) {
const id = typeof target === 'string'
? target
@@ -105,7 +93,6 @@ export {
generateRandomString,
getFormattedObjects,
getSecureTorrentName,
getServerActor,
getServerCommit,
generateVideoImportTmpPath,
getUUIDFromFilename


+ 39
- 3
server/helpers/video.ts View File

@@ -1,14 +1,21 @@
import { VideoModel } from '../models/video/video'
import * as Bluebird from 'bluebird'
import {
isStreamingPlaylist,
MStreamingPlaylistVideo,
MVideo,
MVideoAccountLightBlacklistAllFiles,
MVideoFile,
MVideoFullLight,
MVideoIdThumbnail,
MVideoImmutable,
MVideoThumbnail,
MVideoWithRights,
MVideoImmutable
MVideoWithRights
} from '@server/typings/models'
import { Response } from 'express'
import { DEFAULT_AUDIO_RESOLUTION } from '@server/initializers/constants'
import { JobQueue } from '@server/lib/job-queue'
import { VideoTranscodingPayload } from '@shared/models'

type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' | 'only-immutable-attributes'

@@ -62,10 +69,39 @@ function getVideoWithAttributes (res: Response) {
return res.locals.videoAll || res.locals.onlyVideo || res.locals.onlyVideoWithRights
}

function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile) {
let dataInput: VideoTranscodingPayload

if (videoFile.isAudio()) {
dataInput = {
type: 'merge-audio' as 'merge-audio',
resolution: DEFAULT_AUDIO_RESOLUTION,
videoUUID: video.uuid,
isNewVideo: true
}
} else {
dataInput = {
type: 'optimize' as 'optimize',
videoUUID: video.uuid,
isNewVideo: true
}
}

return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput })
}

function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
return isStreamingPlaylist(videoOrPlaylist)
? videoOrPlaylist.Video
: videoOrPlaylist
}

export {
VideoFetchType,
VideoFetchByUrlType,
fetchVideo,
getVideoWithAttributes,
fetchVideoByUrl
fetchVideoByUrl,
addOptimizeOrMergeAudioJob,
extractVideo
}

+ 1
- 1
server/helpers/webtorrent.ts View File

@@ -13,8 +13,8 @@ import { WEBSERVER } from '@server/initializers/constants'
import * as parseTorrent from 'parse-torrent'
import * as magnetUtil from 'magnet-uri'
import { isArray } from '@server/helpers/custom-validators/misc'
import { extractVideo } from '@server/lib/videos'
import { getTorrentFileName, getVideoFilePath } from '@server/lib/video-paths'
import { extractVideo } from '@server/helpers/video'

const createTorrentPromise = promisify2<string, any, any>(createTorrent)



+ 1
- 2
server/initializers/checker-after-init.ts View File

@@ -1,12 +1,11 @@
import * as config from 'config'
import { isProdInstance, isTestInstance } from '../helpers/core-utils'
import { UserModel } from '../models/account/user'
import { ApplicationModel } from '../models/application/application'
import { getServerActor, ApplicationModel } from '../models/application/application'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { URL } from 'url'
import { CONFIG, isEmailEnabled } from './config'
import { logger } from '../helpers/logger'
import { getServerActor } from '../helpers/utils'
import { RecentlyAddedStrategy } from '../../shared/models/redundancy'
import { isArray } from '../helpers/custom-validators/misc'
import { uniq } from 'lodash'


+ 1
- 1
server/lib/activitypub/actor.ts View File

@@ -19,7 +19,6 @@ import { AvatarModel } from '../../models/avatar/avatar'
import { ServerModel } from '../../models/server/server'
import { VideoChannelModel } from '../../models/video/video-channel'
import { JobQueue } from '../job-queue'
import { getServerActor } from '../../helpers/utils'
import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor'
import { sequelizeTypescript } from '../../initializers/database'
import {
@@ -36,6 +35,7 @@ import {
MChannel
} from '../../typings/models'
import { extname } from 'path'
import { getServerActor } from '@server/models/application/application'

// Set account keys, this could be long so process after the account creation and do not block the client
function setAsyncActorKeys <T extends MActor> (actor: T) {


+ 1
- 1
server/lib/activitypub/follow.ts View File

@@ -3,8 +3,8 @@ import { CONFIG } from '../../initializers/config'
import { SERVER_ACTOR_NAME } from '../../initializers/constants'
import { JobQueue } from '../job-queue'
import { logger } from '../../helpers/logger'
import { getServerActor } from '../../helpers/utils'
import { ServerModel } from '../../models/server/server'
import { getServerActor } from '@server/models/application/application'

async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors) {
if (!CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED) return


+ 0
- 9
server/lib/activitypub/index.ts View File

@@ -1,9 +0,0 @@
export * from './process'
export * from './send'
export * from './actor'
export * from './share'
export * from './playlist'
export * from './videos'
export * from './video-comments'
export * from './video-rates'
export * from './url'

+ 1
- 1
server/lib/activitypub/process/process-follow.ts View File

@@ -7,11 +7,11 @@ import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { sendAccept, sendReject } from '../send'
import { Notifier } from '../../notifier'
import { getAPId } from '../../../helpers/activitypub'
import { getServerActor } from '../../../helpers/utils'
import { CONFIG } from '../../../initializers/config'
import { APProcessorOptions } from '../../../typings/activitypub-processor.model'
import { MActorFollowActors, MActorSignature } from '../../../typings/models'
import { autoFollowBackIfNeeded } from '../follow'
import { getServerActor } from '@server/models/application/application'

async function processFollowActivity (options: APProcessorOptions<ActivityFollow>) {
const { activity, byActor } = options


+ 1
- 1
server/lib/activitypub/send/send-create.ts View File

@@ -6,7 +6,6 @@ import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unic
import { audiencify, getActorsInvolvedInVideo, getAudience, getAudienceFromFollowersOf, getVideoCommentAudience } from '../audience'
import { logger } from '../../../helpers/logger'
import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
import { getServerActor } from '../../../helpers/utils'
import {
MActorLight,
MCommentOwnerVideo,
@@ -17,6 +16,7 @@ import {
MVideoRedundancyStreamingPlaylistVideo
} from '../../../typings/models'
import { ContextType } from '@server/helpers/activitypub'
import { getServerActor } from '@server/models/application/application'

async function sendCreateVideo (video: MVideoAP, t: Transaction) {
if (!video.hasPrivacyForFederation()) return undefined


+ 1
- 1
server/lib/activitypub/send/send-delete.ts View File

@@ -7,9 +7,9 @@ import { getDeleteActivityPubUrl } from '../url'
import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './utils'
import { audiencify, getActorsInvolvedInVideo, getVideoCommentAudience } from '../audience'
import { logger } from '../../../helpers/logger'
import { getServerActor } from '../../../helpers/utils'
import { MCommentOwnerVideoReply, MVideoAccountLight, MVideoPlaylistFullSummary } from '../../../typings/models/video'
import { MActorUrl } from '../../../typings/models'
import { getServerActor } from '@server/models/application/application'

async function sendDeleteVideo (video: MVideoAccountLight, transaction: Transaction) {
logger.info('Creating job to broadcast delete of video %s.', video.url)


+ 1
- 1
server/lib/activitypub/send/send-update.ts View File

@@ -9,7 +9,6 @@ import { broadcastToFollowers, sendVideoRelatedActivity } from './utils'
import { audiencify, getActorsInvolvedInVideo, getAudience } from '../audience'
import { logger } from '../../../helpers/logger'
import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
import { getServerActor } from '../../../helpers/utils'
import {
MAccountDefault,
MActor,
@@ -20,6 +19,7 @@ import {
MVideoPlaylistFull,
MVideoRedundancyVideo
} from '../../../typings/models'
import { getServerActor } from '@server/models/application/application'

async function sendUpdateVideo (videoArg: MVideoAPWithoutCaption, t: Transaction, overrodeByActor?: MActor) {
const video = videoArg as MVideoAP


+ 1
- 1
server/lib/activitypub/send/utils.ts View File

@@ -5,10 +5,10 @@ import { ActorModel } from '../../../models/activitypub/actor'
import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { JobQueue } from '../../job-queue'
import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getRemoteVideoAudience } from '../audience'
import { getServerActor } from '../../../helpers/utils'
import { afterCommitIfTransaction } from '../../../helpers/database-utils'
import { MActor, MActorId, MActorLight, MActorWithInboxes, MVideoAccountLight, MVideoId, MVideoImmutable } from '../../../typings/models'
import { ContextType } from '@server/helpers/activitypub'
import { getServerActor } from '@server/models/application/application'

async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: {
byActor: MActorLight


+ 1
- 1
server/lib/activitypub/share.ts View File

@@ -1,5 +1,4 @@
import { Transaction } from 'sequelize'
import { getServerActor } from '../../helpers/utils'
import { VideoShareModel } from '../../models/video/video-share'
import { sendUndoAnnounce, sendVideoAnnounce } from './send'
import { getVideoAnnounceActivityPubUrl } from './url'
@@ -10,6 +9,7 @@ import { logger } from '../../helpers/logger'
import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub'
import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../typings/models/video'
import { getServerActor } from '@server/models/application/application'

async function shareVideoByServerAndChannel (video: MVideoAccountLight, t: Transaction) {
if (!video.hasPrivacyForFederation()) return undefined


+ 1
- 2
server/lib/activitypub/videos.ts View File

@@ -6,7 +6,7 @@ import {
ActivityHashTagObject,
ActivityMagnetUrlObject,
ActivityPlaylistSegmentHashesObject,
ActivityPlaylistUrlObject,
ActivityPlaylistUrlObject, ActivitypubHttpFetcherPayload,
ActivityTagObject,
ActivityUrlObject,
ActivityVideoUrlObject,
@@ -38,7 +38,6 @@ import { sendCreateVideo, sendUpdateVideo } from './send'
import { isArray } from '../../helpers/custom-validators/misc'
import { VideoCaptionModel } from '../../models/video/video-caption'
import { JobQueue } from '../job-queue'
import { ActivitypubHttpFetcherPayload } from '../job-queue/handlers/activitypub-http-fetcher'
import { createRates } from './video-rates'
import { addVideoShares, shareVideoByServerAndChannel } from './share'
import { fetchVideoByUrl, VideoFetchByUrlType } from '../../helpers/video'


+ 1
- 1
server/lib/avatar.ts View File

@@ -1,7 +1,7 @@
import 'multer'
import { sendUpdateActor } from './activitypub/send'
import { AVATARS_SIZE, LRU_CACHE, QUEUE_CONCURRENCY } from '../initializers/constants'
import { updateActorAvatarInstance } from './activitypub'
import { updateActorAvatarInstance } from './activitypub/actor'
import { processImage } from '../helpers/image-utils'
import { extname, join } from 'path'
import { retryTransactionWrapper } from '../helpers/database-utils'


+ 2
- 12
server/lib/emailer.ts View File

@@ -3,7 +3,6 @@ import { isTestInstance } from '../helpers/core-utils'
import { bunyanLogger, logger } from '../helpers/logger'
import { CONFIG, isEmailEnabled } from '../initializers/config'
import { JobQueue } from './job-queue'
import { EmailPayload } from './job-queue/handlers/email'
import { readFileSync } from 'fs-extra'
import { WEBSERVER } from '../initializers/constants'
import {
@@ -16,15 +15,7 @@ import {
} from '../typings/models/video'
import { MActorFollowActors, MActorFollowFull, MUser } from '../typings/models'
import { MVideoImport, MVideoImportVideo } from '@server/typings/models/video/video-import'

type SendEmailOptions = {
to: string[]
subject: string
text: string

fromDisplayName?: string
replyTo?: string
}
import { EmailPayload } from '@shared/models'

class Emailer {

@@ -507,6 +498,5 @@ class Emailer {
// ---------------------------------------------------------------------------

export {
Emailer,
SendEmailOptions
Emailer
}

+ 1
- 8
server/lib/job-queue/handlers/activitypub-follow.ts View File

@@ -11,14 +11,7 @@ import { ActorModel } from '../../../models/activitypub/actor'
import { Notifier } from '../../notifier'
import { sequelizeTypescript } from '../../../initializers/database'
import { MActor, MActorFollowActors, MActorFull } from '../../../typings/models'

export type ActivitypubFollowPayload = {
followerActorId: number
name: string
host: string
isAutoFollow?: boolean
assertIsChannel?: boolean
}
import { ActivitypubFollowPayload } from '@shared/models'

async function processActivityPubFollow (job: Bull.Job) {
const payload = job.data as ActivitypubFollowPayload


+ 1
- 8
server/lib/job-queue/handlers/activitypub-http-broadcast.ts View File

@@ -5,14 +5,7 @@ import { doRequest } from '../../../helpers/requests'
import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
import { BROADCAST_CONCURRENCY, JOB_REQUEST_TIMEOUT } from '../../../initializers/constants'
import { ActorFollowScoreCache } from '../../files-cache'
import { ContextType } from '@server/helpers/activitypub'

export type ActivitypubHttpBroadcastPayload = {
uris: string[]
signatureActorId?: number
body: any
contextType?: ContextType
}
import { ActivitypubHttpBroadcastPayload } from '@shared/models'

async function processActivityPubHttpBroadcast (job: Bull.Job) {
logger.info('Processing ActivityPub broadcast in job %d.', job.id)


+ 3
- 10
server/lib/job-queue/handlers/activitypub-http-fetcher.ts View File

@@ -5,22 +5,15 @@ import { processActivities } from '../../activitypub/process'
import { addVideoComments } from '../../activitypub/video-comments'
import { crawlCollectionPage } from '../../activitypub/crawl'
import { VideoModel } from '../../../models/video/video'
import { addVideoShares, createRates } from '../../activitypub'
import { addVideoShares } from '../../activitypub/share'
import { createRates } from '../../activitypub/video-rates'
import { createAccountPlaylists } from '../../activitypub/playlist'
import { AccountModel } from '../../../models/account/account'
import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
import { VideoShareModel } from '../../../models/video/video-share'
import { VideoCommentModel } from '../../../models/video/video-comment'
import { MAccountDefault, MVideoFullLight } from '../../../typings/models'

type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' | 'account-playlists'

export type ActivitypubHttpFetcherPayload = {
uri: string
type: FetchType
videoId?: number
accountId?: number
}
import { ActivitypubHttpFetcherPayload, FetchType } from '@shared/models'

async function processActivityPubHttpFetcher (job: Bull.Job) {
logger.info('Processing ActivityPub fetcher in job %d.', job.id)


+ 1
- 8
server/lib/job-queue/handlers/activitypub-http-unicast.ts View File

@@ -4,14 +4,7 @@ import { doRequest } from '../../../helpers/requests'
import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
import { JOB_REQUEST_TIMEOUT } from '../../../initializers/constants'
import { ActorFollowScoreCache } from '../../files-cache'
import { ContextType } from '@server/helpers/activitypub'

export type ActivitypubHttpUnicastPayload = {
uri: string
signatureActorId?: number
body: any
contextType?: ContextType
}
import { ActivitypubHttpUnicastPayload } from '@shared/models'

async function processActivityPubHttpUnicast (job: Bull.Job) {
logger.info('Processing ActivityPub unicast in job %d.', job.id)


+ 4
- 6
server/lib/job-queue/handlers/activitypub-refresher.ts View File

@@ -1,14 +1,12 @@
import * as Bull from 'bull'
import { logger } from '../../../helpers/logger'
import { fetchVideoByUrl } from '../../../helpers/video'
import { refreshActorIfNeeded, refreshVideoIfNeeded, refreshVideoPlaylistIfNeeded } from '../../activitypub'
import { refreshActorIfNeeded } from '../../activitypub/actor'
import { refreshVideoIfNeeded } from '../../activitypub/videos'
import { ActorModel } from '../../../models/activitypub/actor'
import { VideoPlaylistModel } from '../../../models/video/video-playlist'

export type RefreshPayload = {
type: 'video' | 'video-playlist' | 'actor'
url: string
}
import { RefreshPayload } from '@shared/models'
import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlist'

async function refreshAPObject (job: Bull.Job) {
const payload = job.data as RefreshPayload


+ 2
- 3
server/lib/job-queue/handlers/email.ts View File

@@ -1,8 +1,7 @@
import * as Bull from 'bull'
import { logger } from '../../../helpers/logger'
import { Emailer, SendEmailOptions } from '../../emailer'

export type EmailPayload = SendEmailOptions
import { Emailer } from '../../emailer'
import { EmailPayload } from '@shared/models'

async function processEmail (job: Bull.Job) {
const payload = job.data as EmailPayload


+ 2
- 9
server/lib/job-queue/handlers/utils/activitypub-http-utils.ts View File

@@ -1,9 +1,9 @@
import { buildSignedActivity, ContextType } from '../../../../helpers/activitypub'
import { getServerActor } from '../../../../helpers/utils'
import { ActorModel } from '../../../../models/activitypub/actor'
import { sha256 } from '../../../../helpers/core-utils'
import { ACTIVITY_PUB, HTTP_SIGNATURE } from '../../../../initializers/constants'
import { MActor } from '../../../../typings/models'
import { getServerActor } from '@server/models/application/application'
import { buildDigest } from '@server/helpers/peertube-crypto'

type Payload = { body: any, contextType?: ContextType, signatureActorId?: number }

@@ -48,14 +48,7 @@ function buildGlobalHeaders (body: any) {
}
}

function buildDigest (body: any) {
const rawBody = typeof body === 'string' ? body : JSON.stringify(body)

return 'SHA-256=' + sha256(rawBody, 'base64')
}

export {
buildDigest,
buildGlobalHeaders,
computeBody,
buildSignedRequestOptions


+ 1
- 5
server/lib/job-queue/handlers/video-file-import.ts View File

@@ -9,11 +9,7 @@ import { extname } from 'path'
import { MVideoFile, MVideoWithFile } from '@server/typings/models'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { getVideoFilePath } from '@server/lib/video-paths'

export type VideoFileImportPayload = {
videoUUID: string
filePath: string
}
import { VideoFileImportPayload } from '@shared/models'

async function processVideoFileImport (job: Bull.Job) {
const payload = job.data as VideoFileImportPayload


+ 3
- 20
server/lib/job-queue/handlers/video-import.ts View File

@@ -7,8 +7,8 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro
import { extname } from 'path'
import { VideoFileModel } from '../../../models/video/video-file'
import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
import { VideoState } from '../../../../shared'
import { federateVideoIfNeeded } from '../../activitypub'
import { VideoImportPayload, VideoImportTorrentPayload, VideoImportYoutubeDLPayload, VideoState } from '../../../../shared'
import { federateVideoIfNeeded } from '../../activitypub/videos'
import { VideoModel } from '../../../models/video/video'
import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
import { getSecureTorrentName } from '../../../helpers/utils'
@@ -21,24 +21,7 @@ import { ThumbnailType } from '../../../../shared/models/videos/thumbnail.type'
import { MThumbnail } from '../../../typings/models/video/thumbnail'
import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/typings/models/video/video-import'
import { getVideoFilePath } from '@server/lib/video-paths'
import { addOptimizeOrMergeAudioJob } from '@server/lib/videos'

type VideoImportYoutubeDLPayload = {
type: 'youtube-dl'
videoImportId: number

generateThumbnail: boolean
generatePreview: boolean

fileExt?: string
}

type VideoImportTorrentPayload = {
type: 'magnet-uri' | 'torrent-file'
videoImportId: number
}

export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
import { addOptimizeOrMergeAudioJob } from '@server/helpers/video'

async function processVideoImport (job: Bull.Job) {
const payload = job.data as VideoImportPayload


+ 1
- 4
server/lib/job-queue/handlers/video-redundancy.ts View File

@@ -1,10 +1,7 @@
import * as Bull from 'bull'
import { logger } from '../../../helpers/logger'
import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler'

export type VideoRedundancyPayload = {
videoId: number
}
import { VideoRedundancyPayload } from '@shared/models'

async function processVideoRedundancy (job: Bull.Job) {
const payload = job.data as VideoRedundancyPayload


+ 7
- 35
server/lib/job-queue/handlers/video-transcoding.ts View File

@@ -1,9 +1,14 @@
import * as Bull from 'bull'
import { VideoResolution } from '../../../../shared'
import {
MergeAudioTranscodingPayload,
NewResolutionTranscodingPayload,
OptimizeTranscodingPayload,
VideoTranscodingPayload
} from '../../../../shared'
import { logger } from '../../../helpers/logger'
import { VideoModel } from '../../../models/video/video'
import { JobQueue } from '../job-queue'
import { federateVideoIfNeeded } from '../../activitypub'
import { federateVideoIfNeeded } from '../../activitypub/videos'
import { retryTransactionWrapper } from '../../../helpers/database-utils'
import { sequelizeTypescript } from '../../../initializers'
import { computeResolutionsToTranscode } from '../../../helpers/ffmpeg-utils'
@@ -12,39 +17,6 @@ import { Notifier } from '../../notifier'
import { CONFIG } from '../../../initializers/config'
import { MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/typings/models'

interface BaseTranscodingPayload {
videoUUID: string
isNewVideo?: boolean
}

interface HLSTranscodingPayload extends BaseTranscodingPayload {
type: 'hls'
isPortraitMode?: boolean
resolution: VideoResolution
copyCodecs: boolean
}

interface NewResolutionTranscodingPayload extends BaseTranscodingPayload {
type: 'new-resolution'
isPortraitMode?: boolean
resolution: VideoResolution
}

interface MergeAudioTranscodingPayload extends BaseTranscodingPayload {
type: 'merge-audio'
resolution: VideoResolution
}

interface OptimizeTranscodingPayload extends BaseTranscodingPayload {
type: 'optimize'
}

export type VideoTranscodingPayload =
HLSTranscodingPayload
| NewResolutionTranscodingPayload
| OptimizeTranscodingPayload
| MergeAudioTranscodingPayload

async function processVideoTranscoding (job: Bull.Job) {
const payload = job.data as VideoTranscodingPayload
logger.info('Processing video file in job %d.', job.id)


+ 1
- 1
server/lib/job-queue/handlers/video-views.ts View File

@@ -3,7 +3,7 @@ import { logger } from '../../../helpers/logger'
import { VideoModel } from '../../../models/video/video'
import { VideoViewModel } from '../../../models/video/video-views'
import { isTestInstance } from '../../../helpers/core-utils'
import { federateVideoIfNeeded } from '../../activitypub'
import { federateVideoIfNeeded } from '../../activitypub/videos'

async function processVideosViews () {
const lastHour = new Date()


+ 17
- 11
server/lib/job-queue/job-queue.ts View File

@@ -1,19 +1,25 @@
import * as Bull from 'bull'
import { JobState, JobType } from '../../../shared/models'
import {
ActivitypubFollowPayload,
ActivitypubHttpBroadcastPayload,
ActivitypubHttpFetcherPayload, ActivitypubHttpUnicastPayload, EmailPayload,
JobState,
JobType, RefreshPayload, VideoFileImportPayload, VideoImportPayload, VideoRedundancyPayload, VideoTranscodingPayload
} from '../../../shared/models'
import { logger } from '../../helpers/logger'
import { Redis } from '../redis'
import { JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants'
import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast'
import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher'
import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast'
import { EmailPayload, processEmail } from './handlers/email'
import { processVideoTranscoding, VideoTranscodingPayload } from './handlers/video-transcoding'
import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
import { processVideoImport, VideoImportPayload } from './handlers/video-import'
import { processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast'
import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher'
import { processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast'
import { processEmail } from './handlers/email'
import { processVideoTranscoding} from './handlers/video-transcoding'
import { processActivityPubFollow } from './handlers/activitypub-follow'
import { processVideoImport} from './handlers/video-import'
import { processVideosViews } from './handlers/video-views'
import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher'
import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import'
import { processVideoRedundancy, VideoRedundancyPayload } from '@server/lib/job-queue/handlers/video-redundancy'
import { refreshAPObject} from './handlers/activitypub-refresher'
import { processVideoFileImport} from './handlers/video-file-import'
import { processVideoRedundancy} from '@server/lib/job-queue/handlers/video-redundancy'

type CreateJobArgument =
{ type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |


+ 1
- 1
server/lib/notifier.ts View File

@@ -26,7 +26,7 @@ import {
import { MAccountDefault, MActorFollowFull } from '../typings/models'
import { MVideoImportVideo } from '@server/typings/models/video/video-import'
import { ServerBlocklistModel } from '@server/models/server/server-blocklist'
import { getServerActor } from '@server/helpers/utils'
import { getServerActor } from '@server/models/application/application'

class Notifier {



+ 1
- 1
server/lib/redundancy.ts View File

@@ -1,12 +1,12 @@
import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
import { sendUndoCacheFile } from './activitypub/send'
import { Transaction } from 'sequelize'
import { getServerActor } from '../helpers/utils'
import { MActorSignature, MVideoRedundancyVideo } from '@server/typings/models'
import { CONFIG } from '@server/initializers/config'
import { logger } from '@server/helpers/logger'
import { ActorFollowModel } from '@server/models/activitypub/actor-follow'
import { Activity } from '@shared/models'
import { getServerActor } from '@server/models/application/application'

async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) {
const serverActor = await getServerActor()


+ 1
- 1
server/lib/schedulers/auto-follow-index-instances.ts View File

@@ -6,7 +6,7 @@ import { chunk } from 'lodash'
import { doRequest } from '@server/helpers/requests'
import { ActorFollowModel } from '@server/models/activitypub/actor-follow'
import { JobQueue } from '@server/lib/job-queue'
import { getServerActor } from '@server/helpers/utils'
import { getServerActor } from '@server/models/application/application'

export class AutoFollowIndexInstances extends AbstractScheduler {



+ 1
- 1
server/lib/schedulers/update-videos-scheduler.ts View File

@@ -2,7 +2,7 @@ import { logger } from '../../helpers/logger'
import { AbstractScheduler } from './abstract-scheduler'
import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update'
import { retryTransactionWrapper } from '../../helpers/database-utils'
import { federateVideoIfNeeded } from '../activitypub'
import { federateVideoIfNeeded } from '../activitypub/videos'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { Notifier } from '../notifier'
import { sequelizeTypescript } from '../../initializers/database'


+ 2
- 2
server/lib/schedulers/videos-redundancy-scheduler.ts View File

@@ -6,11 +6,10 @@ import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent'
import { join } from 'path'
import { move } from 'fs-extra'
import { getServerActor } from '../../helpers/utils'
import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
import { getVideoCacheFileActivityPubUrl, getVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
import { removeVideoRedundancy } from '../redundancy'
import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
import { getOrCreateVideoAndAccountAndChannel } from '../activitypub/videos'
import { downloadPlaylistSegments } from '../hls'
import { CONFIG } from '../../initializers/config'
import {
@@ -26,6 +25,7 @@ import {
} from '@server/typings/models'
import { getVideoFilename } from '../video-paths'
import { VideoModel } from '@server/models/video/video'
import { getServerActor } from '@server/models/application/application'

type CandidateToDuplicate = {
redundancy: VideosRedundancyStrategy


+ 1
- 1
server/lib/video-comment.ts View File

@@ -2,7 +2,7 @@ import * as Sequelize from 'sequelize'
import { ResultList } from '../../shared/models'
import { VideoCommentThreadTree } from '../../shared/models/videos/video-comment.model'
import { VideoCommentModel } from '../models/video/video-comment'
import { getVideoCommentActivityPubUrl } from './activitypub'
import { getVideoCommentActivityPubUrl } from './activitypub/url'
import { sendCreateVideoComment } from './activitypub/send'
import { MAccountDefault, MComment, MCommentOwnerVideoReply, MVideoFullLight } from '../typings/models'



+ 1
- 1
server/lib/video-paths.ts View File

@@ -1,8 +1,8 @@
import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/typings/models'
import { extractVideo } from './videos'
import { join } from 'path'
import { CONFIG } from '@server/initializers/config'
import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
import { extractVideo } from '@server/helpers/video'

// ################## Video file name ##################



+ 1
- 1
server/lib/video-playlist.ts View File

@@ -1,7 +1,7 @@
import * as Sequelize from 'sequelize'
import { VideoPlaylistModel } from '../models/video/video-playlist'
import { VideoPlaylistPrivacy } from '../../shared/models/videos/playlist/video-playlist-privacy.model'
import { getVideoPlaylistActivityPubUrl } from './activitypub'
import { getVideoPlaylistActivityPubUrl } from './activitypub/url'
import { VideoPlaylistType } from '../../shared/models/videos/playlist/video-playlist-type.model'
import { MAccount } from '../typings/models'
import { MVideoPlaylistOwner } from '../typings/models/video/video-playlist'


+ 0
- 36
server/lib/videos.ts View File

@@ -1,36 +0,0 @@
import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/typings/models'
import { VideoTranscodingPayload } from '@server/lib/job-queue/handlers/video-transcoding'
import { DEFAULT_AUDIO_RESOLUTION } from '@server/initializers/constants'
import { JobQueue } from '@server/lib/job-queue'

function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
return isStreamingPlaylist(videoOrPlaylist)
? videoOrPlaylist.Video
: videoOrPlaylist
}

function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile) {
let dataInput: VideoTranscodingPayload

if (videoFile.isAudio()) {
dataInput = {
type: 'merge-audio' as 'merge-audio',
resolution: DEFAULT_AUDIO_RESOLUTION,
videoUUID: video.uuid,
isNewVideo: true
}
} else {
dataInput = {
type: 'optimize' as 'optimize',
videoUUID: video.uuid,
isNewVideo: true
}
}

return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput })
}

export {
addOptimizeOrMergeAudioJob,
extractVideo
}

+ 1
- 1
server/middlewares/activitypub.ts View File

@@ -3,7 +3,7 @@ import { ActivityDelete, ActivityPubSignature } from '../../shared'
import { logger } from '../helpers/logger'
import { isHTTPSignatureVerified, isJsonLDSignatureVerified, parseHTTPSignature } from '../helpers/peertube-crypto'
import { ACCEPT_HEADERS, ACTIVITY_PUB, HTTP_SIGNATURE } from '../initializers/constants'
import { getOrCreateActorAndServerAndModel } from '../lib/activitypub'
import { getOrCreateActorAndServerAndModel } from '../lib/activitypub/actor'
import { loadActorUrlOrGetFromWebfinger } from '../helpers/webfinger'
import { isActorDeleteActivityValid } from '@server/helpers/custom-validators/activitypub/actor'
import { getAPId } from '@server/helpers/activitypub'


+ 1
- 1
server/middlewares/validators/activitypub/activity.ts View File

@@ -1,7 +1,7 @@
import * as express from 'express'
import { isRootActivityValid } from '../../../helpers/custom-validators/activitypub/activity'
import { logger } from '../../../helpers/logger'
import { getServerActor } from '../../../helpers/utils'
import { getServerActor } from '@server/models/application/application'

async function activityPubValidator (req: express.Request, res: express.Response, next: express.NextFunction) {
logger.debug('Checking activity pub parameters')


+ 1
- 1
server/middlewares/validators/blocklist.ts View File

@@ -6,9 +6,9 @@ import { AccountBlocklistModel } from '../../models/account/account-blocklist'
import { isHostValid } from '../../helpers/custom-validators/servers'
import { ServerBlocklistModel } from '../../models/server/server-blocklist'
import { ServerModel } from '../../models/server/server'
import { getServerActor } from '../../helpers/utils'
import { WEBSERVER } from '../../initializers/constants'
import { doesAccountNameWithHostExist } from '../../helpers/middlewares'
import { getServerActor } from '@server/models/application/application'

const blockAccountValidator = [
body('accountName').exists().withMessage('Should have an account name with host'),


+ 1
- 1
server/middlewares/validators/follows.ts View File

@@ -3,7 +3,6 @@ import { body, param, query } from 'express-validator'
import { isTestInstance } from '../../helpers/core-utils'
import { isEachUniqueHostValid, isHostValid } from '../../helpers/custom-validators/servers'
import { logger } from '../../helpers/logger'
import { getServerActor } from '../../helpers/utils'
import { SERVER_ACTOR_NAME, WEBSERVER } from '../../initializers/constants'
import { ActorFollowModel } from '../../models/activitypub/actor-follow'
import { areValidationErrors } from './utils'
@@ -12,6 +11,7 @@ import { loadActorUrlOrGetFromWebfinger } from '../../helpers/webfinger'
import { isActorTypeValid, isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor'
import { MActorFollowActorsDefault } from '@server/typings/models'
import { isFollowStateValid } from '@server/helpers/custom-validators/follows'
import { getServerActor } from '@server/models/application/application'

const listFollowsValidator = [
query('state')


+ 1
- 1
server/middlewares/validators/videos/videos.ts View File

@@ -38,7 +38,6 @@ import { checkUserCanTerminateOwnershipChange, doesChangeVideoOwnershipExist } f
import { VideoChangeOwnershipAccept } from '../../../../shared/models/videos/video-change-ownership-accept.model'
import { AccountModel } from '../../../models/account/account'
import { isNSFWQueryValid, isNumberArray, isStringArray } from '../../../helpers/custom-validators/search'
import { getServerActor } from '../../../helpers/utils'
import { CONFIG } from '../../../initializers/config'
import { isLocalVideoAccepted } from '../../../lib/moderation'
import { Hooks } from '../../../lib/plugins/hooks'
@@ -50,6 +49,7 @@ import {
} from '../../../helpers/middlewares'
import { MVideoFullLight } from '@server/typings/models'
import { getVideoWithAttributes } from '../../../helpers/video'
import { getServerActor } from '@server/models/application/application'

const videosAddValidator = getCommonVideoEditAttributes().concat([
body('videofile')


+ 1
- 1
server/models/activitypub/actor-follow.ts View File

@@ -20,7 +20,6 @@ import {
import { FollowState } from '../../../shared/models/actors'
import { ActorFollow } from '../../../shared/models/actors/follow.model'
import { logger } from '../../helpers/logger'
import { getServerActor } from '../../helpers/utils'
import { ACTOR_FOLLOW_SCORE, FOLLOW_STATES, SERVER_ACTOR_NAME } from '../../initializers/constants'
import { ServerModel } from '../server/server'
import { createSafeIn, getFollowsSort, getSort } from '../utils'
@@ -37,6 +36,7 @@ import {
} from '@server/typings/models'
import { ActivityPubActorType } from '@shared/models'
import { VideoModel } from '@server/models/video/video'
import { getServerActor } from '@server/models/application/application'

@Table({
tableName: 'actorFollow',


+ 11
- 0
server/models/application/application.ts View File

@@ -1,5 +1,16 @@
import { AllowNull, Column, Default, DefaultScope, HasOne, IsInt, Model, Table } from 'sequelize-typescript'
import { AccountModel } from '../account/account'
import * as memoizee from 'memoizee'

export const getServerActor = memoizee(async function () {
const application = await ApplicationModel.load()
if (!application) throw Error('Could not load Application from database.')

const actor = application.Account.Actor
actor.Account = application.Account

return actor
}, { promise: true })

@DefaultScope(() => ({
include: [


+ 1
- 1
server/models/redundancy/video-redundancy.ts View File

@@ -17,7 +17,6 @@ import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '..
import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
import { VideoFileModel } from '../video/video-file'
import { getServerActor } from '../../helpers/utils'
import { VideoModel } from '../video/video'
import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
import { logger } from '../../helpers/logger'
@@ -37,6 +36,7 @@ import {
StreamingPlaylistRedundancyInformation,
VideoRedundancy
} from '@shared/models/redundancy/video-redundancy.model'
import { getServerActor } from '@server/models/application/application'

export enum ScopeNames {
WITH_VIDEO = 'WITH_VIDEO'


+ 1
- 1
server/models/video/video-comment.ts View File

@@ -9,7 +9,6 @@ import { ActorModel } from '../activitypub/actor'
import { buildBlockedAccountSQL, buildLocalAccountIdsIn, getCommentSort, throwIfNotValid } from '../utils'
import { VideoModel } from './video'
import { VideoChannelModel } from './video-channel'
import { getServerActor } from '../../helpers/utils'
import { actorNameAlphabet } from '../../helpers/custom-validators/activitypub/actor'
import { regexpCapture } from '../../helpers/regexp'
import { uniq } from 'lodash'
@@ -28,6 +27,7 @@ import {
} from '../../typings/models/video'
import { MUserAccountId } from '@server/typings/models'
import { VideoPrivacy } from '@shared/models'
import { getServerActor } from '@server/models/application/application'

enum ScopeNames {
WITH_ACCOUNT = 'WITH_ACCOUNT',


+ 2
- 2
server/models/video/video-format-utils.ts View File

@@ -8,7 +8,7 @@ import {
getVideoDislikesActivityPubUrl,
getVideoLikesActivityPubUrl,
getVideoSharesActivityPubUrl
} from '../../lib/activitypub'
} from '../../lib/activitypub/url'
import { isArray } from '../../helpers/custom-validators/misc'
import { VideoStreamingPlaylist } from '../../../shared/models/videos/video-streaming-playlist.model'
import {
@@ -23,7 +23,7 @@ import {
import { MVideoFileRedundanciesOpt } from '../../typings/models/video/video-file'
import { VideoFile } from '@shared/models/videos/video-file.model'
import { generateMagnetUri } from '@server/helpers/webtorrent'
import { extractVideo } from '@server/lib/videos'
import { extractVideo } from '@server/helpers/video'

export type VideoFormattingJSONOptions = {
completeDescription?: boolean


+ 1
- 1
server/models/video/video.ts View File

@@ -43,7 +43,6 @@ import {
} from '../../helpers/custom-validators/videos'
import { getVideoFileResolution } from '../../helpers/ffmpeg-utils'
import { logger } from '../../helpers/logger'
import { getServerActor } from '../../helpers/utils'
import {
ACTIVITY_PUB,
API_VERSION,
@@ -126,6 +125,7 @@ import { getHLSDirectory, getTorrentFileName, getTorrentFilePath, getVideoFilena
import { ModelCache } from '@server/models/model-cache'
import { buildListQuery, BuildVideosQueryOptions, wrapForAPIResults } from './video-query-builder'
import { buildNSFWFilter } from '@server/helpers/express-utils'
import { getServerActor } from '@server/models/application/application'

export enum ScopeNames {
AVAILABLE_FOR_LIST_IDS = 'AVAILABLE_FOR_LIST_IDS',


+ 2
- 1
server/tests/api/activitypub/security.ts View File

@@ -4,10 +4,11 @@ import 'mocha'

import { cleanupTests, closeAllSequelize, flushAndRunMultipleServers, ServerInfo, setActorField } from '../../../../shared/extra-utils'
import { HTTP_SIGNATURE } from '../../../initializers/constants'
import { buildDigest, buildGlobalHeaders } from '../../../lib/job-queue/handlers/utils/activitypub-http-utils'
import { buildGlobalHeaders } from '../../../lib/job-queue/handlers/utils/activitypub-http-utils'
import * as chai from 'chai'
import { activityPubContextify, buildSignedActivity } from '../../../helpers/activitypub'
import { makeFollowRequest, makePOSTAPRequest } from '../../../../shared/extra-utils/requests/activitypub'
import { buildDigest } from '@server/helpers/peertube-crypto'

const expect = chai.expect



+ 8
- 0
shared/models/server/emailer.model.ts View File

@@ -0,0 +1,8 @@
export type SendEmailOptions = {
to: string[]
subject: string
text: string

fromDisplayName?: string
replyTo?: string
}

+ 1
- 0
shared/models/server/index.ts View File

@@ -2,6 +2,7 @@ export * from './about.model'
export * from './contact-form.model'
export * from './custom-config.model'
export * from './debug.model'
export * from './emailer.model'
export * from './job.model'
export * from './server-config.model'
export * from './server-stats.model'

+ 100
- 0
shared/models/server/job.model.ts View File

@@ -1,3 +1,7 @@
import { ContextType } from '@server/helpers/activitypub'
import { SendEmailOptions } from './emailer.model'
import { VideoResolution } from '@shared/models'

export type JobState = 'active' | 'completed' | 'failed' | 'waiting' | 'delayed'

export type JobType =
@@ -23,3 +27,99 @@ export interface Job {
finishedOn: Date | string
processedOn: Date | string
}

export type ActivitypubHttpBroadcastPayload = {
uris: string[]
signatureActorId?: number
body: any
contextType?: ContextType
}

export type ActivitypubFollowPayload = {
followerActorId: number
name: string
host: string
isAutoFollow?: boolean
assertIsChannel?: boolean
}

export type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' | 'account-playlists'
export type ActivitypubHttpFetcherPayload = {
uri: string
type: FetchType
videoId?: number
accountId?: number
}

export type ActivitypubHttpUnicastPayload = {
uri: string
signatureActorId?: number
body: any
contextType?: ContextType
}

export type RefreshPayload = {
type: 'video' | 'video-playlist' | 'actor'
url: string
}

export type EmailPayload = SendEmailOptions

export type VideoFileImportPayload = {
videoUUID: string
filePath: string
}

export type VideoImportYoutubeDLPayload = {
type: 'youtube-dl'
videoImportId: number

generateThumbnail: boolean
generatePreview: boolean

fileExt?: string
}
export type VideoImportTorrentPayload = {
type: 'magnet-uri' | 'torrent-file'
videoImportId: number
}
export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload

export type VideoRedundancyPayload = {
videoId: number
}

// Video transcoding payloads

interface BaseTranscodingPayload {
videoUUID: string
isNewVideo?: boolean
}

interface HLSTranscodingPayload extends BaseTranscodingPayload {
type: 'hls'
isPortraitMode?: boolean
resolution: VideoResolution
copyCodecs: boolean
}

export interface NewResolutionTranscodingPayload extends BaseTranscodingPayload {
type: 'new-resolution'
isPortraitMode?: boolean
resolution: VideoResolution
}

export interface MergeAudioTranscodingPayload extends BaseTranscodingPayload {
type: 'merge-audio'
resolution: VideoResolution
}

export interface OptimizeTranscodingPayload extends BaseTranscodingPayload {
type: 'optimize'
}

export type VideoTranscodingPayload =
HLSTranscodingPayload
| NewResolutionTranscodingPayload
| OptimizeTranscodingPayload
| MergeAudioTranscodingPayload

Loading…
Cancel
Save