Browse Source

Add ability to list redundancies

tags/contain
Chocobozzz Chocobozzz 3 years ago
parent
commit
b764380ac2
64 changed files with 1806 additions and 194 deletions
  1. +1
    -0
      client/package.json
  2. +1
    -1
      client/src/app/+admin/admin.component.html
  3. +9
    -4
      client/src/app/+admin/admin.module.ts
  4. +4
    -2
      client/src/app/+admin/follows/follows.component.html
  5. +5
    -0
      client/src/app/+admin/follows/follows.routes.ts
  6. +1
    -0
      client/src/app/+admin/follows/index.ts
  7. +1
    -1
      client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts
  8. +0
    -28
      client/src/app/+admin/follows/shared/redundancy.service.ts
  9. +1
    -0
      client/src/app/+admin/follows/video-redundancies-list/index.ts
  10. +82
    -0
      client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html
  11. +37
    -0
      client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss
  12. +178
    -0
      client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts
  13. +24
    -0
      client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html
  14. +8
    -0
      client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss
  15. +11
    -0
      client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts
  16. +8
    -7
      client/src/app/+admin/system/jobs/jobs.component.ts
  17. +7
    -0
      client/src/app/core/server/server.service.ts
  18. +0
    -1
      client/src/app/shared/images/global-icon.component.ts
  19. +4
    -19
      client/src/app/shared/instance/instance-statistics.component.ts
  20. +2
    -0
      client/src/app/shared/shared.module.ts
  21. +73
    -0
      client/src/app/shared/video/redundancy.service.ts
  22. +27
    -1
      client/src/app/shared/video/video-actions-dropdown.component.ts
  23. +2
    -1
      client/src/app/shared/video/video-miniature.component.ts
  24. +4
    -0
      client/src/app/shared/video/video.model.ts
  25. +34
    -1
      client/yarn.lock
  26. +4
    -4
      config/test.yaml
  27. +1
    -1
      package.json
  28. +2
    -2
      scripts/dev/index.sh
  29. +2
    -2
      server/controllers/api/server/follows.ts
  30. +80
    -4
      server/controllers/api/server/redundancy.ts
  31. +9
    -1
      server/controllers/api/server/stats.ts
  32. +1
    -1
      server/helpers/custom-validators/activitypub/cache-file.ts
  33. +12
    -0
      server/helpers/custom-validators/video-redundancies.ts
  34. +2
    -2
      server/helpers/webtorrent.ts
  35. +2
    -2
      server/initializers/config.ts
  36. +13
    -12
      server/initializers/constants.ts
  37. +27
    -0
      server/initializers/migrations/0475-redundancy-expires-on.ts
  38. +2
    -2
      server/lib/activitypub/cache-file.ts
  39. +20
    -0
      server/lib/job-queue/handlers/video-redundancy.ts
  40. +8
    -5
      server/lib/job-queue/job-queue.ts
  41. +4
    -4
      server/lib/redundancy.ts
  42. +0
    -1
      server/lib/schedulers/update-videos-scheduler.ts
  43. +46
    -13
      server/lib/schedulers/videos-redundancy-scheduler.ts
  44. +14
    -9
      server/middlewares/sort.ts
  45. +71
    -3
      server/middlewares/validators/redundancy.ts
  46. +3
    -0
      server/middlewares/validators/sort.ts
  47. +177
    -9
      server/models/redundancy/video-redundancy.ts
  48. +135
    -6
      server/tests/api/check-params/redundancy.ts
  49. +1
    -0
      server/tests/api/redundancy/index.ts
  50. +373
    -0
      server/tests/api/redundancy/manage-redundancy.ts
  51. +115
    -23
      server/tests/api/redundancy/redundancy.ts
  52. +4
    -1
      server/typings/models/video/video-file.ts
  53. +5
    -1
      server/typings/models/video/video-streaming-playlist.ts
  54. +11
    -2
      server/typings/models/video/video.ts
  55. +67
    -3
      shared/extra-utils/server/redundancy.ts
  56. +16
    -2
      shared/extra-utils/videos/videos.ts
  57. +3
    -1
      shared/models/redundancy/index.ts
  58. +1
    -0
      shared/models/redundancy/video-redundancies-filters.model.ts
  59. +33
    -0
      shared/models/redundancy/video-redundancy.model.ts
  60. +2
    -1
      shared/models/redundancy/videos-redundancy-strategy.model.ts
  61. +2
    -1
      shared/models/server/job.model.ts
  62. +10
    -8
      shared/models/server/server-stats.model.ts
  63. +3
    -1
      shared/models/users/user-right.enum.ts
  64. +1
    -1
      shared/models/videos/video.model.ts

+ 1
- 0
client/package.json View File

@@ -77,6 +77,7 @@
"bootstrap": "^4.1.3",
"buffer": "^5.1.0",
"cache-chunk-store": "^3.0.0",
"chart.js": "^2.9.3",
"codelyzer": "^5.0.1",
"core-js": "^3.1.4",
"css-loader": "^3.1.0",


+ 1
- 1
client/src/app/+admin/admin.component.html View File

@@ -5,7 +5,7 @@
</a>

<a i18n *ngIf="hasServerFollowRight()" routerLink="/admin/follows" routerLinkActive="active" class="title-page">
Manage follows
Follows & redundancies
</a>

<a i18n *ngIf="hasVideoAbusesRight() || hasVideoBlacklistRight()" routerLink="/admin/moderation" routerLinkActive="active" class="title-page">


+ 9
- 4
client/src/app/+admin/admin.module.ts View File

@@ -5,7 +5,7 @@ import { TableModule } from 'primeng/table'
import { SharedModule } from '../shared'
import { AdminRoutingModule } from './admin-routing.module'
import { AdminComponent } from './admin.component'
import { FollowersListComponent, FollowingAddComponent, FollowsComponent } from './follows'
import { FollowersListComponent, FollowingAddComponent, FollowsComponent, VideoRedundanciesListComponent } from './follows'
import { FollowingListComponent } from './follows/following-list/following-list.component'
import { UserCreateComponent, UserListComponent, UserPasswordComponent, UsersComponent, UserUpdateComponent } from './users'
import {
@@ -16,7 +16,6 @@ import {
} from './moderation'
import { ModerationComponent } from '@app/+admin/moderation/moderation.component'
import { RedundancyCheckboxComponent } from '@app/+admin/follows/shared/redundancy-checkbox.component'
import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service'
import { InstanceAccountBlocklistComponent, InstanceServerBlocklistComponent } from '@app/+admin/moderation/instance-blocklist'
import { JobsComponent } from '@app/+admin/system/jobs/jobs.component'
import { JobService, LogsComponent, LogsService, SystemComponent } from '@app/+admin/system'
@@ -27,13 +26,18 @@ import { PluginSearchComponent } from '@app/+admin/plugins/plugin-search/plugin-
import { PluginShowInstalledComponent } from '@app/+admin/plugins/plugin-show-installed/plugin-show-installed.component'
import { SelectButtonModule } from 'primeng/selectbutton'
import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service'
import { VideoRedundancyInformationComponent } from '@app/+admin/follows/video-redundancies-list/video-redundancy-information.component'
import { ChartModule } from 'primeng/chart'

@NgModule({
imports: [
AdminRoutingModule,

SharedModule,

TableModule,
SelectButtonModule,
SharedModule
ChartModule
],

declarations: [
@@ -44,6 +48,8 @@ import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service'
FollowersListComponent,
FollowingListComponent,
RedundancyCheckboxComponent,
VideoRedundanciesListComponent,
VideoRedundancyInformationComponent,

UsersComponent,
UserCreateComponent,
@@ -78,7 +84,6 @@ import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service'
],

providers: [
RedundancyService,
JobService,
LogsService,
DebugService,


+ 4
- 2
client/src/app/+admin/follows/follows.component.html View File

@@ -1,5 +1,5 @@
<div class="admin-sub-header">
<div i18n class="form-sub-title">Manage follows</div>
<div i18n class="form-sub-title">Follows & redundancies</div>

<div class="admin-sub-nav">
<a i18n routerLink="following-list" routerLinkActive="active">Following</a>
@@ -7,7 +7,9 @@
<a i18n routerLink="following-add" routerLinkActive="active">Follow</a>

<a i18n routerLink="followers-list" routerLinkActive="active">Followers</a>

<a i18n routerLink="video-redundancies-list" routerLinkActive="active">Video redundancies</a>
</div>
</div>

<router-outlet></router-outlet>
<router-outlet></router-outlet>

+ 5
- 0
client/src/app/+admin/follows/follows.routes.ts View File

@@ -6,6 +6,7 @@ import { FollowingAddComponent } from './following-add'
import { FollowersListComponent } from './followers-list'
import { UserRight } from '../../../../../shared'
import { FollowingListComponent } from './following-list/following-list.component'
import { VideoRedundanciesListComponent } from '@app/+admin/follows/video-redundancies-list'

export const FollowsRoutes: Routes = [
{
@@ -47,6 +48,10 @@ export const FollowsRoutes: Routes = [
title: 'Add follow'
}
}
},
{
path: 'video-redundancies-list',
component: VideoRedundanciesListComponent
}
]
}


+ 1
- 0
client/src/app/+admin/follows/index.ts View File

@@ -1,5 +1,6 @@
export * from './following-add'
export * from './followers-list'
export * from './following-list'
export * from './video-redundancies-list'
export * from './follows.component'
export * from './follows.routes'

+ 1
- 1
client/src/app/+admin/follows/shared/redundancy-checkbox.component.ts View File

@@ -1,7 +1,7 @@
import { Component, Input } from '@angular/core'
import { Notifier } from '@app/core'
import { I18n } from '@ngx-translate/i18n-polyfill'
import { RedundancyService } from '@app/+admin/follows/shared/redundancy.service'
import { RedundancyService } from '@app/shared/video/redundancy.service'

@Component({
selector: 'my-redundancy-checkbox',


+ 0
- 28
client/src/app/+admin/follows/shared/redundancy.service.ts View File

@@ -1,28 +0,0 @@
import { catchError, map } from 'rxjs/operators'
import { HttpClient } from '@angular/common/http'
import { Injectable } from '@angular/core'
import { RestExtractor } from '@app/shared'
import { environment } from '../../../../environments/environment'

@Injectable()
export class RedundancyService {
static BASE_USER_SUBSCRIPTIONS_URL = environment.apiUrl + '/api/v1/server/redundancy'

constructor (
private authHttp: HttpClient,
private restExtractor: RestExtractor
) { }

updateRedundancy (host: string, redundancyAllowed: boolean) {
const url = RedundancyService.BASE_USER_SUBSCRIPTIONS_URL + '/' + host

const body = { redundancyAllowed }

return this.authHttp.put(url, body)
.pipe(
map(this.restExtractor.extractDataBool),
catchError(err => this.restExtractor.handleError(err))
)
}

}

+ 1
- 0
client/src/app/+admin/follows/video-redundancies-list/index.ts View File

@@ -0,0 +1 @@
export * from './video-redundancies-list.component'

+ 82
- 0
client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.html View File

@@ -0,0 +1,82 @@
<div class="admin-sub-header">
<div i18n class="form-sub-title">Video redundancies list</div>

<div class="select-filter-block">
<label for="displayType" i18n>Display</label>

<div class="peertube-select-container">
<select id="displayType" name="displayType" [(ngModel)]="displayType" (ngModelChange)="onDisplayTypeChanged()">
<option value="my-videos">My videos duplicated by remote instances</option>
<option value="remote-videos">Remote videos duplicated by my instance</option>
</select>
</div>
</div>
</div>

<p-table
[value]="videoRedundancies" [lazy]="true" [paginator]="true" [totalRecords]="totalRecords" [rows]="rowsPerPage"
[sortField]="sort.field" [sortOrder]="sort.order" (onLazyLoad)="loadLazy($event)" dataKey="id"
>
<ng-template pTemplate="header">
<tr>
<th i18n *ngIf="isDisplayingRemoteVideos()">Strategy</th>
<th i18n pSortableColumn="name">Video name <p-sortIcon field="name"></p-sortIcon></th>
<th i18n>Video URL</th>
<th i18n *ngIf="isDisplayingRemoteVideos()">Total size</th>
<th></th>
</tr>
</ng-template>

<ng-template pTemplate="body" let-redundancy>
<tr class="expander" [pRowToggler]="redundancy">
<td *ngIf="isDisplayingRemoteVideos()">{{ getRedundancyStrategy(redundancy) }}</td>

<td>{{ redundancy.name }}</td>

<td>
<a target="_blank" rel="noopener noreferrer" [href]="redundancy.url">{{ redundancy.url }}</a>
</td>

<td *ngIf="isDisplayingRemoteVideos()">{{ getTotalSize(redundancy) | bytes: 1 }}</td>

<td class="action-cell">
<my-delete-button (click)="removeRedundancy(redundancy)"></my-delete-button>
</td>
</tr>
</ng-template>

<ng-template pTemplate="rowexpansion" let-redundancy>
<tr>
<td colspan="2">
<div *ngFor="let file of redundancy.redundancies.files" class="expansion-block">
<my-video-redundancy-information [redundancyElement]="file"></my-video-redundancy-information>
</div>
</td>
</tr>

<tr>
<td colspan="2">
<div *ngFor="let playlist of redundancy.redundancies.streamingPlaylists">
<my-video-redundancy-information [redundancyElement]="playlist"></my-video-redundancy-information>
</div>
</td>
</tr>
</ng-template>
</p-table>


<div class="redundancies-charts" *ngIf="isDisplayingRemoteVideos()">
<div class="form-sub-title" i18n>Enabled strategies stats</div>

<div class="chart-blocks">

<div *ngIf="noRedundancies" i18n class="no-results">
No redundancy strategy is enabled on your instance.
</div>

<div class="chart-block" *ngFor="let r of redundanciesGraphsData">
<p-chart type="pie" [data]="r.graphData" [options]="r.options" width="300px" height="300px"></p-chart>
</div>

</div>
</div>

+ 37
- 0
client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.scss View File

@@ -0,0 +1,37 @@
@import '_variables';
@import '_mixins';

.expansion-block {
margin-bottom: 20px;
}

.admin-sub-header {
align-items: flex-end;

.select-filter-block {
&:not(:last-child) {
margin-right: 10px;
}

label {
margin-bottom: 2px;
}

.peertube-select-container {
@include peertube-select-container(auto);
}
}
}

.redundancies-charts {
margin-top: 50px;

.chart-blocks {
display: flex;
justify-content: center;

.chart-block {
margin: 0 20px;
}
}
}

+ 178
- 0
client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts View File

@@ -0,0 +1,178 @@
import { Component, OnInit } from '@angular/core'
import { Notifier, ServerService } from '@app/core'
import { SortMeta } from 'primeng/api'
import { ConfirmService } from '../../../core/confirm/confirm.service'
import { RestPagination, RestTable } from '../../../shared'
import { I18n } from '@ngx-translate/i18n-polyfill'
import { VideoRedundanciesTarget, VideoRedundancy } from '@shared/models'
import { peertubeLocalStorage } from '@app/shared/misc/peertube-web-storage'
import { VideosRedundancyStats } from '@shared/models/server'
import { BytesPipe } from 'ngx-pipes'
import { RedundancyService } from '@app/shared/video/redundancy.service'

@Component({
selector: 'my-video-redundancies-list',
templateUrl: './video-redundancies-list.component.html',
styleUrls: [ './video-redundancies-list.component.scss' ]
})
export class VideoRedundanciesListComponent extends RestTable implements OnInit {
private static LOCAL_STORAGE_DISPLAY_TYPE = 'video-redundancies-list-display-type'

videoRedundancies: VideoRedundancy[] = []
totalRecords = 0
rowsPerPage = 10

sort: SortMeta = { field: 'name', order: 1 }
pagination: RestPagination = { count: this.rowsPerPage, start: 0 }
displayType: VideoRedundanciesTarget = 'my-videos'

redundanciesGraphsData: { stats: VideosRedundancyStats, graphData: object, options: object }[] = []

noRedundancies = false

private bytesPipe: BytesPipe

constructor (
private notifier: Notifier,
private confirmService: ConfirmService,
private redundancyService: RedundancyService,
private serverService: ServerService,
private i18n: I18n
) {
super()

this.bytesPipe = new BytesPipe()
}

ngOnInit () {
this.loadSelectLocalStorage()

this.initialize()

this.serverService.getServerStats()
.subscribe(res => {
const redundancies = res.videosRedundancy

if (redundancies.length === 0) this.noRedundancies = true

for (const r of redundancies) {
this.buildPieData(r)
}
})
}

isDisplayingRemoteVideos () {
return this.displayType === 'remote-videos'
}

getTotalSize (redundancy: VideoRedundancy) {
return redundancy.redundancies.files.reduce((a, b) => a + b.size, 0) +
redundancy.redundancies.streamingPlaylists.reduce((a, b) => a + b.size, 0)
}

onDisplayTypeChanged () {
this.pagination.start = 0
this.saveSelectLocalStorage()

this.loadData()
}

getRedundancyStrategy (redundancy: VideoRedundancy) {
if (redundancy.redundancies.files.length !== 0) return redundancy.redundancies.files[0].strategy
if (redundancy.redundancies.streamingPlaylists.length !== 0) return redundancy.redundancies.streamingPlaylists[0].strategy

return ''
}

buildPieData (stats: VideosRedundancyStats) {
const totalSize = stats.totalSize
? stats.totalSize - stats.totalUsed
: stats.totalUsed

if (totalSize === 0) return

this.redundanciesGraphsData.push({
stats,
graphData: {
labels: [ this.i18n('Used'), this.i18n('Available') ],
datasets: [
{
data: [ stats.totalUsed, totalSize ],
backgroundColor: [
'#FF6384',
'#36A2EB'
],
hoverBackgroundColor: [
'#FF6384',
'#36A2EB'
]
}
]
},
options: {
title: {
display: true,
text: stats.strategy
},

tooltips: {
callbacks: {
label: (tooltipItem: any, data: any) => {
const dataset = data.datasets[tooltipItem.datasetIndex]
let label = data.labels[tooltipItem.index]
if (label) label += ': '
else label = ''

label += this.bytesPipe.transform(dataset.data[tooltipItem.index], 1)
return label
}
}
}
}
})
}

async removeRedundancy (redundancy: VideoRedundancy) {
const message = this.i18n('Do you really want to remove this video redundancy?')
const res = await this.confirmService.confirm(message, this.i18n('Remove redundancy'))
if (res === false) return

this.redundancyService.removeVideoRedundancies(redundancy)
.subscribe(
() => {
this.notifier.success(this.i18n('Video redundancies removed!'))
this.loadData()
},

err => this.notifier.error(err.message)
)

}

protected loadData () {
const options = {
pagination: this.pagination,
sort: this.sort,
target: this.displayType
}

this.redundancyService.listVideoRedundancies(options)
.subscribe(
resultList => {
this.videoRedundancies = resultList.data
this.totalRecords = resultList.total
},

err => this.notifier.error(err.message)
)
}

private loadSelectLocalStorage () {
const displayType = peertubeLocalStorage.getItem(VideoRedundanciesListComponent.LOCAL_STORAGE_DISPLAY_TYPE)
if (displayType) this.displayType = displayType as VideoRedundanciesTarget
}

private saveSelectLocalStorage () {
peertubeLocalStorage.setItem(VideoRedundanciesListComponent.LOCAL_STORAGE_DISPLAY_TYPE, this.displayType)
}
}

+ 24
- 0
client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.html View File

@@ -0,0 +1,24 @@
<div>
<span class="label">Url</span>
<a target="_blank" rel="noopener noreferrer" [href]="redundancyElement.fileUrl">{{ redundancyElement.fileUrl }}</a>
</div>

<div>
<span class="label">Created on</span>
<span>{{ redundancyElement.createdAt | date: 'medium' }}</span>
</div>

<div>
<span class="label">Expires on</span>
<span>{{ redundancyElement.expiresOn | date: 'medium' }}</span>
</div>

<div>
<span class="label">Size</span>
<span>{{ redundancyElement.size | bytes: 1 }}</span>
</div>

<div *ngIf="redundancyElement.strategy">
<span class="label">Strategy</span>
<span>{{ redundancyElement.strategy }}</span>
</div>

+ 8
- 0
client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.scss View File

@@ -0,0 +1,8 @@
@import '_variables';
@import '_mixins';

.label {
display: inline-block;
min-width: 100px;
font-weight: $font-semibold;
}

+ 11
- 0
client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts View File

@@ -0,0 +1,11 @@
import { Component, Input } from '@angular/core'
import { FileRedundancyInformation, StreamingPlaylistRedundancyInformation } from '@shared/models'

@Component({
selector: 'my-video-redundancy-information',
templateUrl: './video-redundancy-information.component.html',
styleUrls: [ './video-redundancy-information.component.scss' ]
})
export class VideoRedundancyInformationComponent {
@Input() redundancyElement: FileRedundancyInformation | StreamingPlaylistRedundancyInformation
}

+ 8
- 7
client/src/app/+admin/system/jobs/jobs.component.ts View File

@@ -16,8 +16,8 @@ import { JobTypeClient } from '../../../../types/job-type-client.type'
styleUrls: [ './jobs.component.scss' ]
})
export class JobsComponent extends RestTable implements OnInit {
private static JOB_STATE_LOCAL_STORAGE_STATE = 'jobs-list-state'
private static JOB_STATE_LOCAL_STORAGE_TYPE = 'jobs-list-type'
private static LOCAL_STORAGE_STATE = 'jobs-list-state'
private static LOCAL_STORAGE_TYPE = 'jobs-list-type'

jobState: JobStateClient = 'waiting'
jobStates: JobStateClient[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed' ]
@@ -34,7 +34,8 @@ export class JobsComponent extends RestTable implements OnInit {
'video-file-import',
'video-import',
'videos-views',
'activitypub-refresher'
'activitypub-refresher',
'video-redundancy'
]

jobs: Job[] = []
@@ -77,15 +78,15 @@ export class JobsComponent extends RestTable implements OnInit {
}

private loadJobStateAndType () {
const state = peertubeLocalStorage.getItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE)
const state = peertubeLocalStorage.getItem(JobsComponent.LOCAL_STORAGE_STATE)
if (state) this.jobState = state as JobState

const type = peertubeLocalStorage.getItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_TYPE)
const type = peertubeLocalStorage.getItem(JobsComponent.LOCAL_STORAGE_TYPE)
if (type) this.jobType = type as JobType
}

private saveJobStateAndType () {
peertubeLocalStorage.setItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_STATE, this.jobState)
peertubeLocalStorage.setItem(JobsComponent.JOB_STATE_LOCAL_STORAGE_TYPE, this.jobType)
peertubeLocalStorage.setItem(JobsComponent.LOCAL_STORAGE_STATE, this.jobState)
peertubeLocalStorage.setItem(JobsComponent.LOCAL_STORAGE_TYPE, this.jobType)
}
}

+ 7
- 0
client/src/app/core/server/server.service.ts View File

@@ -9,6 +9,7 @@ import { VideoConstant } from '../../../../../shared/models/videos'
import { isDefaultLocale, peertubeTranslate } from '../../../../../shared/models/i18n'
import { getDevLocale, isOnDevLocale } from '@app/shared/i18n/i18n-utils'
import { sortBy } from '@app/shared/misc/utils'
import { ServerStats } from '@shared/models/server'

@Injectable()
export class ServerService {
@@ -16,6 +17,8 @@ export class ServerService {
private static BASE_VIDEO_URL = environment.apiUrl + '/api/v1/videos/'
private static BASE_VIDEO_PLAYLIST_URL = environment.apiUrl + '/api/v1/video-playlists/'
private static BASE_LOCALE_URL = environment.apiUrl + '/client/locales/'
private static BASE_STATS_URL = environment.apiUrl + '/api/v1/server/stats'

private static CONFIG_LOCAL_STORAGE_KEY = 'server-config'

configReloaded = new Subject<void>()
@@ -235,6 +238,10 @@ export class ServerService {
return this.localeObservable.pipe(first())
}

getServerStats () {
return this.http.get<ServerStats>(ServerService.BASE_STATS_URL)
}

private loadAttributeEnum <T extends string | number> (
baseUrl: string,
attributeName: 'categories' | 'licences' | 'languages' | 'privacies',


+ 0
- 1
client/src/app/shared/images/global-icon.component.ts View File

@@ -1,6 +1,5 @@
import { ChangeDetectionStrategy, Component, ElementRef, Input, OnInit } from '@angular/core'
import { HooksService } from '@app/core/plugins/hooks.service'
import { I18n } from '@ngx-translate/i18n-polyfill'

const icons = {
'add': require('!!raw-loader?!../../../assets/images/global/add.svg'),


+ 4
- 19
client/src/app/shared/instance/instance-statistics.component.ts View File

@@ -1,9 +1,6 @@
import { map } from 'rxjs/operators'
import { HttpClient } from '@angular/common/http'
import { Component, OnInit } from '@angular/core'
import { I18n } from '@ngx-translate/i18n-polyfill'
import { ServerStats } from '@shared/models/server'
import { environment } from '../../../environments/environment'
import { ServerService } from '@app/core'

@Component({
selector: 'my-instance-statistics',
@@ -11,27 +8,15 @@ import { environment } from '../../../environments/environment'
styleUrls: [ './instance-statistics.component.scss' ]
})
export class InstanceStatisticsComponent implements OnInit {
private static BASE_STATS_URL = environment.apiUrl + '/api/v1/server/stats'

serverStats: ServerStats = null

constructor (
private http: HttpClient,
private i18n: I18n
private serverService: ServerService
) {
}

ngOnInit () {
this.getStats()
.subscribe(
res => {
this.serverStats = res
}
)
}

getStats () {
return this.http
.get<ServerStats>(InstanceStatisticsComponent.BASE_STATS_URL)
this.serverService.getServerStats()
.subscribe(res => this.serverStats = res)
}
}

+ 2
- 0
client/src/app/shared/shared.module.ts View File

@@ -98,6 +98,7 @@ import { FollowService } from '@app/shared/instance/follow.service'
import { MultiSelectModule } from 'primeng/multiselect'
import { FeatureBooleanComponent } from '@app/shared/instance/feature-boolean.component'
import { InputReadonlyCopyComponent } from '@app/shared/forms/input-readonly-copy.component'
import { RedundancyService } from '@app/shared/video/redundancy.service'

@NgModule({
imports: [
@@ -300,6 +301,7 @@ import { InputReadonlyCopyComponent } from '@app/shared/forms/input-readonly-cop
UserNotificationService,

FollowService,
RedundancyService,

I18n
]


+ 73
- 0
client/src/app/shared/video/redundancy.service.ts View File

@@ -0,0 +1,73 @@
import { catchError, map, toArray } from 'rxjs/operators'
import { HttpClient, HttpParams } from '@angular/common/http'
import { Injectable } from '@angular/core'
import { RestExtractor, RestPagination, RestService } from '@app/shared/rest'
import { SortMeta } from 'primeng/api'
import { ResultList, Video, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models'
import { concat, Observable } from 'rxjs'
import { environment } from '../../../environments/environment'

@Injectable()
export class RedundancyService {
static BASE_REDUNDANCY_URL = environment.apiUrl + '/api/v1/server/redundancy'

constructor (
private authHttp: HttpClient,
private restService: RestService,
private restExtractor: RestExtractor
) { }

updateRedundancy (host: string, redundancyAllowed: boolean) {
const url = RedundancyService.BASE_REDUNDANCY_URL + '/' + host

const body = { redundancyAllowed }

return this.authHttp.put(url, body)
.pipe(
map(this.restExtractor.extractDataBool),
catchError(err => this.restExtractor.handleError(err))
)
}

listVideoRedundancies (options: {
pagination: RestPagination,
sort: SortMeta,
target?: VideoRedundanciesTarget
}): Observable<ResultList<VideoRedundancy>> {
const { pagination, sort, target } = options

let params = new HttpParams()
params = this.restService.addRestGetParams(params, pagination, sort)

if (target) params = params.append('target', target)

return this.authHttp.get<ResultList<VideoRedundancy>>(RedundancyService.BASE_REDUNDANCY_URL + '/videos', { params })
.pipe(
catchError(res => this.restExtractor.handleError(res))
)
}

addVideoRedundancy (video: Video) {
return this.authHttp.post(RedundancyService.BASE_REDUNDANCY_URL + '/videos', { videoId: video.id })
.pipe(
catchError(res => this.restExtractor.handleError(res))
)
}

removeVideoRedundancies (redundancy: VideoRedundancy) {
const observables = redundancy.redundancies.streamingPlaylists.map(r => r.id)
.concat(redundancy.redundancies.files.map(r => r.id))
.map(id => this.removeRedundancy(id))

return concat(...observables)
.pipe(toArray())
}

private removeRedundancy (redundancyId: number) {
return this.authHttp.delete(RedundancyService.BASE_REDUNDANCY_URL + '/videos/' + redundancyId)
.pipe(
map(this.restExtractor.extractDataBool),
catchError(res => this.restExtractor.handleError(res))
)
}
}

+ 27
- 1
client/src/app/shared/video/video-actions-dropdown.component.ts View File

@@ -14,6 +14,7 @@ import { VideoBlacklistComponent } from '@app/shared/video/modals/video-blacklis
import { VideoBlacklistService } from '@app/shared/video-blacklist'
import { ScreenService } from '@app/shared/misc/screen.service'
import { VideoCaption } from '@shared/models'
import { RedundancyService } from '@app/shared/video/redundancy.service'

export type VideoActionsDisplayType = {
playlist?: boolean
@@ -22,6 +23,7 @@ export type VideoActionsDisplayType = {
blacklist?: boolean
delete?: boolean
report?: boolean
duplicate?: boolean
}

@Component({
@@ -46,7 +48,8 @@ export class VideoActionsDropdownComponent implements OnChanges {
update: true,
blacklist: true,
delete: true,
report: true
report: true,
duplicate: true
}
@Input() placement = 'left'

@@ -74,6 +77,7 @@ export class VideoActionsDropdownComponent implements OnChanges {
private screenService: ScreenService,
private videoService: VideoService,
private blocklistService: BlocklistService,
private redundancyService: RedundancyService,
private i18n: I18n
) { }

@@ -144,6 +148,10 @@ export class VideoActionsDropdownComponent implements OnChanges {
return this.video && this.video instanceof VideoDetails && this.video.downloadEnabled
}

canVideoBeDuplicated () {
return this.video.canBeDuplicatedBy(this.user)
}

/* Action handlers */

async unblacklistVideo () {
@@ -186,6 +194,18 @@ export class VideoActionsDropdownComponent implements OnChanges {
)
}

duplicateVideo () {
this.redundancyService.addVideoRedundancy(this.video)
.subscribe(
() => {
const message = this.i18n('This video will be duplicated by your instance.')
this.notifier.success(message)
},

err => this.notifier.error(err.message)
)
}

onVideoBlacklisted () {
this.videoBlacklisted.emit()
}
@@ -233,6 +253,12 @@ export class VideoActionsDropdownComponent implements OnChanges {
iconName: 'undo',
isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.blacklist && this.isVideoUnblacklistable()
},
{
label: this.i18n('Duplicate (redundancy)'),
handler: () => this.duplicateVideo(),
isDisplayed: () => this.authService.isLoggedIn() && this.displayOptions.duplicate && this.canVideoBeDuplicated(),
iconName: 'cloud-download'
},
{
label: this.i18n('Delete'),
handler: () => this.removeVideo(),


+ 2
- 1
client/src/app/shared/video/video-miniature.component.ts View File

@@ -64,7 +64,8 @@ export class VideoMiniatureComponent implements OnInit {
update: true,
blacklist: true,
delete: true,
report: true
report: true,
duplicate: false
}
showActions = false
serverConfig: ServerConfig


+ 4
- 0
client/src/app/shared/video/video.model.ts View File

@@ -152,4 +152,8 @@ export class Video implements VideoServerModel {
isUpdatableBy (user: AuthUser) {
return user && this.isLocal === true && (this.account.name === user.username || user.hasRight(UserRight.UPDATE_ANY_VIDEO))
}

canBeDuplicatedBy (user: AuthUser) {
return user && this.isLocal === false && user.hasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES)
}
}

+ 34
- 1
client/yarn.lock View File

@@ -2586,6 +2586,29 @@ chardet@^0.7.0:
resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e"
integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==

chart.js@^2.9.3:
version "2.9.3"
resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-2.9.3.tgz#ae3884114dafd381bc600f5b35a189138aac1ef7"
integrity sha512-+2jlOobSk52c1VU6fzkh3UwqHMdSlgH1xFv9FKMqHiNCpXsGPQa/+81AFa+i3jZ253Mq9aAycPwDjnn1XbRNNw==
dependencies:
chartjs-color "^2.1.0"
moment "^2.10.2"

chartjs-color-string@^0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz#1df096621c0e70720a64f4135ea171d051402f71"
integrity sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A==
dependencies:
color-name "^1.0.0"

chartjs-color@^2.1.0:
version "2.4.1"
resolved "https://registry.yarnpkg.com/chartjs-color/-/chartjs-color-2.4.1.tgz#6118bba202fe1ea79dd7f7c0f9da93467296c3b0"
integrity sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w==
dependencies:
chartjs-color-string "^0.6.0"
color-convert "^1.9.3"

check-types@^8.0.3:
version "8.0.3"
resolved "https://registry.yarnpkg.com/check-types/-/check-types-8.0.3.tgz#3356cca19c889544f2d7a95ed49ce508a0ecf552"
@@ -2800,7 +2823,7 @@ collection-visit@^1.0.0:
map-visit "^1.0.0"
object-visit "^1.0.0"

color-convert@^1.9.0:
color-convert@^1.9.0, color-convert@^1.9.3:
version "1.9.3"
resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==
@@ -2812,6 +2835,11 @@ color-name@1.1.3:
resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25"
integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=

color-name@^1.0.0:
version "1.1.4"
resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==

colors@1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/colors/-/colors-1.1.2.tgz#168a4701756b6a7f51a12ce0c97bfa28c084ed63"
@@ -6941,6 +6969,11 @@ mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1:
dependencies:
minimist "0.0.8"

moment@^2.10.2:
version "2.24.0"
resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b"
integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==

mousetrap@^1.6.0:
version "1.6.3"
resolved "https://registry.yarnpkg.com/mousetrap/-/mousetrap-1.6.3.tgz#80fee49665fd478bccf072c9d46bdf1bfed3558a"


+ 4
- 4
config/test.yaml View File

@@ -40,18 +40,18 @@ contact_form:

redundancy:
videos:
check_interval: '10 minutes'
check_interval: '1 minute'
strategies:
-
size: '10MB'
size: '1000MB'
min_lifetime: '10 minutes'
strategy: 'most-views'
-
size: '10MB'
size: '1000MB'
min_lifetime: '10 minutes'
strategy: 'trending'
-
size: '10MB'
size: '1000MB'
min_lifetime: '10 minutes'
strategy: 'recently-added'
min_views: 1


+ 1
- 1
package.json View File

@@ -41,7 +41,7 @@
"i18n:create-custom-files": "node ./dist/scripts/i18n/create-custom-files.js",
"reset-password": "node ./dist/scripts/reset-password.js",
"play": "scripty",
"dev": "scripty",
"dev": "sh ./scripts/dev/index.sh",
"dev:server": "sh ./scripts/dev/server.sh",
"dev:embed": "scripty",
"dev:client": "sh ./scripts/dev/client.sh",


+ 2
- 2
scripts/dev/index.sh View File

@@ -3,5 +3,5 @@
set -eu

NODE_ENV=test npm run concurrently -- -k \
"npm run dev:client -- --skip-server" \
"npm run dev:server"
"sh scripts/dev/client.sh --skip-server" \
"sh scripts/dev/server.sh"

+ 2
- 2
server/controllers/api/server/follows.ts View File

@@ -24,7 +24,7 @@ import {
} from '../../../middlewares/validators'
import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
import { JobQueue } from '../../../lib/job-queue'
import { removeRedundancyOf } from '../../../lib/redundancy'
import { removeRedundanciesOfServer } from '../../../lib/redundancy'
import { sequelizeTypescript } from '../../../initializers/database'
import { autoFollowBackIfNeeded } from '../../../lib/activitypub/follow'

@@ -153,7 +153,7 @@ async function removeFollowing (req: express.Request, res: express.Response) {
await server.save({ transaction: t })

// Async, could be long
removeRedundancyOf(server.id)
removeRedundanciesOfServer(server.id)
.catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err))

await follow.destroy({ transaction: t })


+ 80
- 4
server/controllers/api/server/redundancy.ts View File

@@ -1,9 +1,24 @@
import * as express from 'express'
import { UserRight } from '../../../../shared/models/users'
import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy'
import { removeRedundancyOf } from '../../../lib/redundancy'
import {
asyncMiddleware,
authenticate,
ensureUserHasRight,
paginationValidator,
setDefaultPagination,
setDefaultVideoRedundanciesSort,
videoRedundanciesSortValidator
} from '../../../middlewares'
import {
listVideoRedundanciesValidator,
updateServerRedundancyValidator,
addVideoRedundancyValidator,
removeVideoRedundancyValidator
} from '../../../middlewares/validators/redundancy'
import { removeRedundanciesOfServer, removeVideoRedundancy } from '../../../lib/redundancy'
import { logger } from '../../../helpers/logger'
import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy'
import { JobQueue } from '@server/lib/job-queue'

const serverRedundancyRouter = express.Router()

@@ -14,6 +29,31 @@ serverRedundancyRouter.put('/redundancy/:host',
asyncMiddleware(updateRedundancy)
)

serverRedundancyRouter.get('/redundancy/videos',
authenticate,
ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES),
listVideoRedundanciesValidator,
paginationValidator,
videoRedundanciesSortValidator,
setDefaultVideoRedundanciesSort,
setDefaultPagination,
asyncMiddleware(listVideoRedundancies)
)

serverRedundancyRouter.post('/redundancy/videos',
authenticate,
ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES),
addVideoRedundancyValidator,
asyncMiddleware(addVideoRedundancy)
)

serverRedundancyRouter.delete('/redundancy/videos/:redundancyId',
authenticate,
ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES),
removeVideoRedundancyValidator,
asyncMiddleware(removeVideoRedundancyController)
)

// ---------------------------------------------------------------------------

export {
@@ -22,6 +62,42 @@ export {

// ---------------------------------------------------------------------------

async function listVideoRedundancies (req: express.Request, res: express.Response) {
const resultList = await VideoRedundancyModel.listForApi({
start: req.query.start,
count: req.query.count,
sort: req.query.sort,
target: req.query.target,
strategy: req.query.strategy
})

const result = {
total: resultList.total,
data: resultList.data.map(r => VideoRedundancyModel.toFormattedJSONStatic(r))
}

return res.json(result)
}

async function addVideoRedundancy (req: express.Request, res: express.Response) {
const payload = {
videoId: res.locals.onlyVideo.id
}

await JobQueue.Instance.createJob({
type: 'video-redundancy',
payload
})

return res.sendStatus(204)
}

async function removeVideoRedundancyController (req: express.Request, res: express.Response) {
await removeVideoRedundancy(res.locals.videoRedundancy)

return res.sendStatus(204)
}

async function updateRedundancy (req: express.Request, res: express.Response) {
const server = res.locals.server

@@ -30,7 +106,7 @@ async function updateRedundancy (req: express.Request, res: express.Response) {
await server.save()

// Async, could be long
removeRedundancyOf(server.id)
removeRedundanciesOfServer(server.id)
.catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err }))

return res.sendStatus(204)


+ 9
- 1
server/controllers/api/server/stats.ts View File

@@ -10,6 +10,7 @@ import { ROUTE_CACHE_LIFETIME } from '../../../initializers/constants'
import { cacheRoute } from '../../../middlewares/cache'
import { VideoFileModel } from '../../../models/video/video-file'
import { CONFIG } from '../../../initializers/config'
import { VideoRedundancyStrategyWithManual } from '@shared/models'

const statsRouter = express.Router()

@@ -25,8 +26,15 @@ async function getStats (req: express.Request, res: express.Response) {
const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
const { totalLocalVideoFilesSize } = await VideoFileModel.getStats()

const strategies: { strategy: VideoRedundancyStrategyWithManual, size: number }[] = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES
.map(r => ({
strategy: r.strategy,
size: r.size
}))
strategies.push({ strategy: 'manual', size: null })

const videosRedundancyStats = await Promise.all(
CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.map(r => {
strategies.map(r => {
return VideoRedundancyModel.getStats(r.strategy)
.then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size }))
})


+ 1
- 1
server/helpers/custom-validators/activitypub/cache-file.ts View File

@@ -6,7 +6,7 @@ import { CacheFileObject } from '../../../../shared/models/activitypub/objects'
function isCacheFileObjectValid (object: CacheFileObject) {
return exists(object) &&
object.type === 'CacheFile' &&
isDateValid(object.expires) &&
(object.expires === null || isDateValid(object.expires)) &&
isActivityPubUrlValid(object.object) &&
(isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url))
}


+ 12
- 0
server/helpers/custom-validators/video-redundancies.ts View File

@@ -0,0 +1,12 @@
import { exists } from './misc'

function isVideoRedundancyTarget (value: any) {
return exists(value) &&
(value === 'my-videos' || value === 'remote-videos')
}

// ---------------------------------------------------------------------------

export {
isVideoRedundancyTarget
}

+ 2
- 2
server/helpers/webtorrent.ts View File

@@ -9,12 +9,12 @@ import { promisify2 } from './core-utils'
import { MVideo } from '@server/typings/models/video/video'
import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/typings/models/video/video-file'
import { isStreamingPlaylist, MStreamingPlaylistVideo } from '@server/typings/models/video/video-streaming-playlist'
import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
import { WEBSERVER } from '@server/initializers/constants'
import * as parseTorrent from 'parse-torrent'
import * as magnetUtil from 'magnet-uri'
import { isArray } from '@server/helpers/custom-validators/misc'
import { extractVideo } from '@server/lib/videos'
import { getTorrentFileName, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
import { getTorrentFileName, getVideoFilePath } from '@server/lib/video-paths'

const createTorrentPromise = promisify2<string, any, any>(createTorrent)



+ 2
- 2
server/initializers/config.ts View File

@@ -1,6 +1,6 @@
import { IConfig } from 'config'
import { dirname, join } from 'path'
import { VideosRedundancy } from '../../shared/models'
import { VideosRedundancyStrategy } from '../../shared/models'
// Do not use barrels, remain constants as independent as possible
import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils'
import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
@@ -304,7 +304,7 @@ function getLocalConfigFilePath () {
return join(dirname(configSources[ 0 ].name), filename + '.json')
}

function buildVideosRedundancy (objs: any[]): VideosRedundancy[] {
function buildVideosRedundancy (objs: any[]): VideosRedundancyStrategy[] {
if (!objs) return []

if (!Array.isArray(objs)) return objs


+ 13
- 12
server/initializers/constants.ts View File

@@ -14,7 +14,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'

// ---------------------------------------------------------------------------

const LAST_MIGRATION_VERSION = 470
const LAST_MIGRATION_VERSION = 475

// ---------------------------------------------------------------------------

@@ -73,7 +73,9 @@ const SORTABLE_COLUMNS = {

PLUGINS: [ 'name', 'createdAt', 'updatedAt' ],

AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ]
AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ],

VIDEO_REDUNDANCIES: [ 'name' ]
}

const OAUTH_LIFETIME = {
@@ -117,45 +119,44 @@ const REMOTE_SCHEME = {
WS: 'wss'
}

// TODO: remove 'video-file'
const JOB_ATTEMPTS: { [id in (JobType | 'video-file')]: number } = {
const JOB_ATTEMPTS: { [id in JobType]: number } = {
'activitypub-http-broadcast': 5,
'activitypub-http-unicast': 5,
'activitypub-http-fetcher': 5,
'activitypub-follow': 5,
'video-file-import': 1,
'video-transcoding': 1,
'video-file': 1,
'video-import': 1,
'email': 5,
'videos-views': 1,
'activitypub-refresher': 1
'activitypub-refresher': 1,
'video-redundancy': 1
}
const JOB_CONCURRENCY: { [id in (JobType | 'video-file')]: number } = {
const JOB_CONCURRENCY: { [id in JobType]: number } = {
'activitypub-http-broadcast': 1,
'activitypub-http-unicast': 5,
'activitypub-http-fetcher': 1,
'activitypub-follow': 1,
'video-file-import': 1,
'video-transcoding': 1,
'video-file': 1,
'video-import': 1,
'email': 5,
'videos-views': 1,
'activitypub-refresher': 1
'activitypub-refresher': 1,
'video-redundancy': 1
}
const JOB_TTL: { [id in (JobType | 'video-file')]: number } = {
const JOB_TTL: { [id in JobType]: number } = {
'activitypub-http-broadcast': 60000 * 10, // 10 minutes
'activitypub-http-unicast': 60000 * 10, // 10 minutes
'activitypub-http-fetcher': 60000 * 10, // 10 minutes
'activitypub-follow': 60000 * 10, // 10 minutes
'video-file-import': 1000 * 3600, // 1 hour
'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long
'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long
'video-import': 1000 * 3600 * 2, // hours
'email': 60000 * 10, // 10 minutes
'videos-views': undefined, // Unlimited
'activitypub-refresher': 60000 * 10 // 10 minutes
'activitypub-refresher': 60000 * 10, // 10 minutes
'video-redundancy': 1000 * 3600 * 3 // 3 hours
}
const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = {
'videos-views': {


+ 27
- 0
server/initializers/migrations/0475-redundancy-expires-on.ts View File

@@ -0,0 +1,27 @@
import * as Sequelize from 'sequelize'

async function up (utils: {
transaction: Sequelize.Transaction,
queryInterface: Sequelize.QueryInterface,
sequelize: Sequelize.Sequelize,
db: any
}): Promise<void> {
{
const data = {
type: Sequelize.DATE,
allowNull: true,
defaultValue: null
}

await utils.queryInterface.changeColumn('videoRedundancy', 'expiresOn', data)
}
}

function down (options) {
throw new Error('Not implemented.')
}

export {
up,
down
}

+ 2
- 2
server/lib/activitypub/cache-file.ts View File

@@ -13,7 +13,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)

return {
expiresOn: new Date(cacheFileObject.expires),
expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
url: cacheFileObject.id,
fileUrl: url.href,
strategy: null,
@@ -30,7 +30,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)

return {
expiresOn: new Date(cacheFileObject.expires),
expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
url: cacheFileObject.id,
fileUrl: url.href,
strategy: null,


+ 20
- 0
server/lib/job-queue/handlers/video-redundancy.ts View File

@@ -0,0 +1,20 @@
import * as Bull from 'bull'
import { logger } from '../../../helpers/logger'
import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler'

export type VideoRedundancyPayload = {
videoId: number
}

async function processVideoRedundancy (job: Bull.Job) {
const payload = job.data as VideoRedundancyPayload
logger.info('Processing video redundancy in job %d.', job.id)

return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId)
}

// ---------------------------------------------------------------------------

export {
processVideoRedundancy
}

+ 8
- 5
server/lib/job-queue/job-queue.ts View File

@@ -13,6 +13,7 @@ import { processVideoImport, VideoImportPayload } from './handlers/video-import'
import { processVideosViews } from './handlers/video-views'
import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher'
import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import'
import { processVideoRedundancy, VideoRedundancyPayload } from '@server/lib/job-queue/handlers/video-redundancy'

type CreateJobArgument =
{ type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -24,20 +25,21 @@ type CreateJobArgument =
{ type: 'email', payload: EmailPayload } |
{ type: 'video-import', payload: VideoImportPayload } |
{ type: 'activitypub-refresher', payload: RefreshPayload } |
{ type: 'videos-views', payload: {} }
{ type: 'videos-views', payload: {} } |
{ type: 'video-redundancy', payload: VideoRedundancyPayload }

const handlers: { [ id in (JobType | 'video-file') ]: (job: Bull.Job) => Promise<any>} = {
const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
'activitypub-http-broadcast': processActivityPubHttpBroadcast,
'activitypub-http-unicast': processActivityPubHttpUnicast,
'activitypub-http-fetcher': processActivityPubHttpFetcher,
'activitypub-follow': processActivityPubFollow,
'video-file-import': processVideoFileImport,
'video-transcoding': processVideoTranscoding,
'video-file': processVideoTranscoding, // TODO: remove it (changed in 1.3)
'email': processEmail,
'video-import': processVideoImport,
'videos-views': processVideosViews,
'activitypub-refresher': refreshAPObject
'activitypub-refresher': refreshAPObject,
'video-redundancy': processVideoRedundancy
}

const jobTypes: JobType[] = [
@@ -50,7 +52,8 @@ const jobTypes: JobType[] = [
'video-file-import',
'video-import',
'videos-views',
'activitypub-refresher'
'activitypub-refresher',
'video-redundancy'
]

class JobQueue {


+ 4
- 4
server/lib/redundancy.ts View File

@@ -13,10 +13,10 @@ async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?
await videoRedundancy.destroy({ transaction: t })
}

async function removeRedundancyOf (serverId: number) {
const videosRedundancy = await VideoRedundancyModel.listLocalOfServer(serverId)
async function removeRedundanciesOfServer (serverId: number) {
const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId)

for (const redundancy of videosRedundancy) {
for (const redundancy of redundancies) {
await removeVideoRedundancy(redundancy)
}
}
@@ -24,6 +24,6 @@ async function removeRedundancyOf (serverId: number) {
// ---------------------------------------------------------------------------

export {
removeRedundancyOf,
removeRedundanciesOfServer,
removeVideoRedundancy
}

+ 0
- 1
server/lib/schedulers/update-videos-scheduler.ts View File

@@ -4,7 +4,6 @@ import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-upda
import { retryTransactionWrapper } from '../../helpers/database-utils'
import { federateVideoIfNeeded } from '../activitypub'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { VideoPrivacy } from '../../../shared/models/videos'
import { Notifier } from '../notifier'
import { sequelizeTypescript } from '../../initializers/database'
import { MVideoFullLight } from '@server/typings/models'


+ 46
- 13
server/lib/schedulers/videos-redundancy-scheduler.ts View File

@@ -1,7 +1,7 @@
import { AbstractScheduler } from './abstract-scheduler'
import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants'
import { logger } from '../../helpers/logger'
import { VideosRedundancy } from '../../../shared/models/redundancy'
import { VideosRedundancyStrategy } from '../../../shared/models/redundancy'
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent'
import { join } from 'path'
@@ -25,9 +25,10 @@ import {
MVideoWithAllFiles
} from '@server/typings/models'
import { getVideoFilename } from '../video-paths'
import { VideoModel } from '@server/models/video/video'

type CandidateToDuplicate = {
redundancy: VideosRedundancy,
redundancy: VideosRedundancyStrategy,
video: MVideoWithAllFiles,
files: MVideoFile[],
streamingPlaylists: MStreamingPlaylistFiles[]
@@ -41,7 +42,7 @@ function isMVideoRedundancyFileVideo (

export class VideosRedundancyScheduler extends AbstractScheduler {

private static instance: AbstractScheduler
private static instance: VideosRedundancyScheduler

protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL

@@ -49,6 +50,22 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
super()
}

async createManualRedundancy (videoId: number) {
const videoToDuplicate = await VideoModel.loadWithFiles(videoId)

if (!videoToDuplicate) {
logger.warn('Video to manually duplicate %d does not exist anymore.', videoId)
return
}

return this.createVideoRedundancies({
video: videoToDuplicate,
redundancy: null,
files: videoToDuplicate.VideoFiles,
streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
})
}

protected async internalExecute () {
for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy)
@@ -94,7 +111,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
for (const redundancyModel of expired) {
try {
const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
const candidate = {
const candidate: CandidateToDuplicate = {
redundancy: redundancyConfig,
video: null,
files: [],
@@ -140,7 +157,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
}
}

private findVideoToDuplicate (cache: VideosRedundancy) {
private findVideoToDuplicate (cache: VideosRedundancyStrategy) {
if (cache.strategy === 'most-views') {
return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
}
@@ -187,13 +204,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
}
}

private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: MVideoAccountLight, fileArg: MVideoFile) {
private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) {
let strategy = 'manual'
let expiresOn: Date = null

if (redundancy) {
strategy = redundancy.strategy
expiresOn = this.buildNewExpiration(redundancy.minLifetime)
}

const file = fileArg as MVideoFileVideo
file.Video = video

const serverActor = await getServerActor()

logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy)

const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs)
@@ -204,10 +229,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
await move(tmpPath, destPath, { overwrite: true })

const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({
expiresOn: this.buildNewExpiration(redundancy.minLifetime),
expiresOn,
url: getVideoCacheFileActivityPubUrl(file),
fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL),
strategy: redundancy.strategy,
strategy,
videoFileId: file.id,
actorId: serverActor.id
})
@@ -220,25 +245,33 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
}

private async createStreamingPlaylistRedundancy (
redundancy: VideosRedundancy,
redundancy: VideosRedundancyStrategy,
video: MVideoAccountLight,
playlistArg: MStreamingPlaylist
) {
let strategy = 'manual'
let expiresOn: Date = null

if (redundancy) {
strategy = redundancy.strategy
expiresOn = this.buildNewExpiration(redundancy.minLifetime)
}

const playlist = playlistArg as MStreamingPlaylistVideo
playlist.Video = video

const serverActor = await getServerActor()

logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy)
logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)

const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)

const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
expiresOn: this.buildNewExpiration(redundancy.minLifetime),
expiresOn,
url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL),
strategy: redundancy.strategy,
strategy,
videoStreamingPlaylistId: playlist.id,
actorId: serverActor.id
})


+ 14
- 9
server/middlewares/sort.ts View File

@@ -1,17 +1,11 @@
import * as express from 'express'
import { SortType } from '../models/utils'

function setDefaultSort (req: express.Request, res: express.Response, next: express.NextFunction) {
if (!req.query.sort) req.query.sort = '-createdAt'

return next()
}
const setDefaultSort = setDefaultSortFactory('-createdAt')

function setDefaultSearchSort (req: express.Request, res: express.Response, next: express.NextFunction) {
if (!req.query.sort) req.query.sort = '-match'
const setDefaultVideoRedundanciesSort = setDefaultSortFactory('name')

return next()
}
const setDefaultSearchSort = setDefaultSortFactory('-match')

function setBlacklistSort (req: express.Request, res: express.Response, next: express.NextFunction) {
let newSort: SortType = { sortModel: undefined, sortValue: '' }
@@ -39,5 +33,16 @@ function setBlacklistSort (req: express.Request, res: express.Response, next: ex
export {
setDefaultSort,
setDefaultSearchSort,
setDefaultVideoRedundanciesSort,
setBlacklistSort
}

// ---------------------------------------------------------------------------

function setDefaultSortFactory (sort: string) {
return (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (!req.query.sort) req.query.sort = sort

return next()
}
}

+ 71
- 3
server/middlewares/validators/redundancy.ts View File

@@ -1,12 +1,13 @@
import * as express from 'express'
import { body, param } from 'express-validator'
import { exists, isBooleanValid, isIdOrUUIDValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc'
import { body, param, query } from 'express-validator'
import { exists, isBooleanValid, isIdOrUUIDValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc'
import { logger } from '../../helpers/logger'
import { areValidationErrors } from './utils'
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
import { isHostValid } from '../../helpers/custom-validators/servers'
import { ServerModel } from '../../models/server/server'
import { doesVideoExist } from '../../helpers/middlewares'
import { isVideoRedundancyTarget } from '@server/helpers/custom-validators/video-redundancies'

const videoFileRedundancyGetValidator = [
param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid video id'),
@@ -101,10 +102,77 @@ const updateServerRedundancyValidator = [
}
]

const listVideoRedundanciesValidator = [
query('target')
.custom(isVideoRedundancyTarget).withMessage('Should have a valid video redundancies target'),

async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking listVideoRedundanciesValidator parameters', { parameters: req.query })

if (areValidationErrors(req, res)) return

return next()
}
]

const addVideoRedundancyValidator = [
body('videoId')
.custom(isIdValid)
.withMessage('Should have a valid video id'),

async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking addVideoRedundancyValidator parameters', { parameters: req.query })

if (areValidationErrors(req, res)) return

if (!await doesVideoExist(req.body.videoId, res, 'only-video')) return

if (res.locals.onlyVideo.remote === false) {
return res.status(400)
.json({ error: 'Cannot create a redundancy on a local video' })
.end()
}

const alreadyExists = await VideoRedundancyModel.isLocalByVideoUUIDExists(res.locals.onlyVideo.uuid)
if (alreadyExists) {
return res.status(409)
.json({ error: 'This video is already duplicated by your instance.' })
}

return next()
}
]

const removeVideoRedundancyValidator = [
param('redundancyId')
.custom(isIdValid)
.withMessage('Should have a valid redundancy id'),

async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking removeVideoRedundancyValidator parameters', { parameters: req.query })

if (areValidationErrors(req, res)) return

const redundancy = await VideoRedundancyModel.loadByIdWithVideo(parseInt(req.params.redundancyId, 10))
if (!redundancy) {
return res.status(404)
.json({ error: 'Video redundancy not found' })
.end()
}

res.locals.videoRedundancy = redundancy

return next()
}
]

// ---------------------------------------------------------------------------

export {
videoFileRedundancyGetValidator,
videoPlaylistRedundancyGetValidator,
updateServerRedundancyValidator
updateServerRedundancyValidator,
listVideoRedundanciesValidator,
addVideoRedundancyValidator,
removeVideoRedundancyValidator
}

+ 3
- 0
server/middlewares/validators/sort.ts View File

@@ -23,6 +23,7 @@ const SORTABLE_USER_NOTIFICATIONS_COLUMNS = createSortableColumns(SORTABLE_COLUM
const SORTABLE_VIDEO_PLAYLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_PLAYLISTS)
const SORTABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.PLUGINS)
const SORTABLE_AVAILABLE_PLUGINS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.AVAILABLE_PLUGINS)
const SORTABLE_VIDEO_REDUNDANCIES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES)

const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS)
const accountsSortValidator = checkSort(SORTABLE_ACCOUNTS_COLUMNS)
@@ -45,6 +46,7 @@ const userNotificationsSortValidator = checkSort(SORTABLE_USER_NOTIFICATIONS_COL
const videoPlaylistsSortValidator = checkSort(SORTABLE_VIDEO_PLAYLISTS_COLUMNS)
const pluginsSortValidator = checkSort(SORTABLE_PLUGINS_COLUMNS)
const availablePluginsSortValidator = checkSort(SORTABLE_AVAILABLE_PLUGINS_COLUMNS)
const videoRedundanciesSortValidator = checkSort(SORTABLE_VIDEO_REDUNDANCIES_COLUMNS)

// ---------------------------------------------------------------------------

@@ -69,5 +71,6 @@ export {
serversBlocklistSortValidator,
userNotificationsSortValidator,
videoPlaylistsSortValidator,
videoRedundanciesSortValidator,
pluginsSortValidator
}

+ 177
- 9
server/models/redundancy/video-redundancy.ts View File

@@ -13,13 +13,13 @@ import {
UpdatedAt
} from 'sequelize-typescript'
import { ActorModel } from '../activitypub/actor'
import { getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../utils'
import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants'
import { VideoFileModel } from '../video/video-file'
import { getServerActor } from '../../helpers/utils'
import { VideoModel } from '../video/video'
import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
import { VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../shared/models/redundancy'
import { logger } from '../../helpers/logger'
import { CacheFileObject, VideoPrivacy } from '../../../shared'
import { VideoChannelModel } from '../video/video-channel'
@@ -27,10 +27,16 @@ import { ServerModel } from '../server/server'
import { sample } from 'lodash'
import { isTestInstance } from '../../helpers/core-utils'
import * as Bluebird from 'bluebird'
import { col, FindOptions, fn, literal, Op, Transaction } from 'sequelize'
import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
import { CONFIG } from '../../initializers/config'
import { MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models'
import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/typings/models'
import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
import {
FileRedundancyInformation,
StreamingPlaylistRedundancyInformation,
VideoRedundancy
} from '@shared/models/redundancy/video-redundancy.model'

export enum ScopeNames {
WITH_VIDEO = 'WITH_VIDEO'
@@ -86,7 +92,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
@UpdatedAt
updatedAt: Date

@AllowNull(false)
@AllowNull(true)
@Column
expiresOn: Date

@@ -193,6 +199,15 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
}

static loadByIdWithVideo (id: number, transaction?: Transaction): Bluebird<MVideoRedundancyVideo> {
const query = {
where: { id },
transaction
}

return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
}

static loadByUrl (url: string, transaction?: Transaction): Bluebird<MVideoRedundancy> {
const query = {
where: {
@@ -394,7 +409,8 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
[Op.ne]: actor.id
},
expiresOn: {
[ Op.lt ]: new Date()
[ Op.lt ]: new Date(),
[ Op.ne ]: null
}
}
}
@@ -447,7 +463,112 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
return VideoRedundancyModel.findAll(query)
}

static async getStats (strategy: VideoRedundancyStrategy) {
static listForApi (options: {
start: number,
count: number,
sort: string,
target: VideoRedundanciesTarget,
strategy?: string
}) {
const { start, count, sort, target, strategy } = options
let redundancyWhere: WhereOptions = {}
let videosWhere: WhereOptions = {}
let redundancySqlSuffix = ''

if (target === 'my-videos') {
Object.assign(videosWhere, { remote: false })
} else if (target === 'remote-videos') {
Object.assign(videosWhere, { remote: true })
Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } })
redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL'
}

if (strategy) {
Object.assign(redundancyWhere, { strategy: strategy })
}

const videoFilterWhere = {
[Op.and]: [
{
[ Op.or ]: [
{
id: {
[ Op.in ]: literal(
'(' +
'SELECT "videoId" FROM "videoFile" ' +
'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' +
redundancySqlSuffix +
')'
)
}
},
{
id: {
[ Op.in ]: literal(
'(' +
'select "videoId" FROM "videoStreamingPlaylist" ' +
'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' +
redundancySqlSuffix +
')'
)
}
}
]
},

videosWhere
]
}

// /!\ On video model /!\
const findOptions = {
offset: start,
limit: count,
order: getSort(sort),
include: [
{
required: false,
model: VideoFileModel.unscoped(),
include: [
{
model: VideoRedundancyModel.unscoped(),
required: false,
where: redundancyWhere
}
]
},
{
required: false,
model: VideoStreamingPlaylistModel.unscoped(),
include: [
{
model: VideoRedundancyModel.unscoped(),
required: false,
where: redundancyWhere
},
{
model: VideoFileModel.unscoped(),
required: false
}
]
}
],
where: videoFilterWhere
}

// /!\ On video model /!\
const countOptions = {
where: videoFilterWhere
}

return Promise.all([
VideoModel.findAll(findOptions),

VideoModel.count(countOptions)
]).then(([ data, total ]) => ({ total, data }))
}

static async getStats (strategy: VideoRedundancyStrategyWithManual) {
const actor = await getServerActor()

const query: FindOptions = {
@@ -478,6 +599,53 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
}))