Browse Source

Add ability to import video with youtube-dl

tags/delete
Chocobozzz 4 years ago
parent
commit
fbad87b047
42 changed files with 1506 additions and 445 deletions
  1. +1
    -0
      client/package.json
  2. +2
    -0
      client/src/app/shared/shared.module.ts
  3. +1
    -0
      client/src/app/shared/video-import/index.ts
  4. +56
    -0
      client/src/app/shared/video-import/video-import.service.ts
  5. +20
    -20
      client/src/app/shared/video/video-edit.model.ts
  6. +10
    -58
      client/src/app/videos/+video-edit/video-add.component.html
  7. +35
    -82
      client/src/app/videos/+video-edit/video-add.component.scss
  8. +15
    -237
      client/src/app/videos/+video-edit/video-add.component.ts
  9. +5
    -1
      client/src/app/videos/+video-edit/video-add.module.ts
  10. +55
    -0
      client/src/app/videos/+video-edit/video-import.component.html
  11. +37
    -0
      client/src/app/videos/+video-edit/video-import.component.scss
  12. +161
    -0
      client/src/app/videos/+video-edit/video-import.component.ts
  13. +0
    -1
      client/src/app/videos/+video-edit/video-update.component.ts
  14. +58
    -0
      client/src/app/videos/+video-edit/video-upload.component.html
  15. +85
    -0
      client/src/app/videos/+video-edit/video-upload.component.scss
  16. +251
    -0
      client/src/app/videos/+video-edit/video-upload.component.ts
  17. +6
    -0
      config/default.yaml
  18. +4
    -3
      package.json
  19. +151
    -0
      server/controllers/api/videos/import.ts
  20. +2
    -1
      server/controllers/api/videos/index.ts
  21. +1
    -1
      server/helpers/custom-validators/activitypub/videos.ts
  22. +30
    -0
      server/helpers/custom-validators/video-imports.ts
  23. +1
    -1
      server/helpers/logger.ts
  24. +142
    -0
      server/helpers/youtube-dl.ts
  25. +16
    -2
      server/initializers/constants.ts
  26. +3
    -1
      server/initializers/database.ts
  27. +129
    -0
      server/lib/job-queue/handlers/video-import.ts
  28. +7
    -3
      server/lib/job-queue/job-queue.ts
  29. +1
    -0
      server/middlewares/validators/index.ts
  30. +51
    -0
      server/middlewares/validators/video-imports.ts
  31. +32
    -30
      server/middlewares/validators/videos.ts
  32. +0
    -1
      server/models/account/account.ts
  33. +105
    -0
      server/models/video/video-import.ts
  34. +1
    -1
      server/models/video/video.ts
  35. +2
    -1
      shared/models/server/job.model.ts
  36. +4
    -0
      shared/models/videos/index.ts
  37. +6
    -0
      shared/models/videos/video-import-create.model.ts
  38. +5
    -0
      shared/models/videos/video-import-state.enum.ts
  39. +5
    -0
      shared/models/videos/video-import-update.model.ts
  40. +7
    -0
      shared/models/videos/video-import.model.ts
  41. +2
    -1
      shared/models/videos/video-state.enum.ts
  42. +1
    -0
      tsconfig.json

+ 1
- 0
client/package.json View File

@@ -23,6 +23,7 @@
"ngx-extractor": "ngx-extractor"
},
"license": "GPLv3",
"typings": "*.d.ts",
"resolutions": {
"video.js": "^7",
"webtorrent/create-torrent/junk": "^1",


+ 2
- 0
client/src/app/shared/shared.module.ts View File

@@ -51,6 +51,7 @@ import { ScreenService } from '@app/shared/misc/screen.service'
import { VideoCaptionsValidatorsService } from '@app/shared/forms/form-validators/video-captions-validators.service'
import { VideoCaptionService } from '@app/shared/video-caption'
import { PeertubeCheckboxComponent } from '@app/shared/forms/peertube-checkbox.component'
import { VideoImportService } from '@app/shared/video-import/video-import.service'

@NgModule({
imports: [
@@ -143,6 +144,7 @@ import { PeertubeCheckboxComponent } from '@app/shared/forms/peertube-checkbox.c
VideoCommentValidatorsService,
VideoValidatorsService,
VideoCaptionsValidatorsService,
VideoImportService,

I18nPrimengCalendarService,
ScreenService,


+ 1
- 0
client/src/app/shared/video-import/index.ts View File

@@ -0,0 +1 @@
export * from './video-import.service'

+ 56
- 0
client/src/app/shared/video-import/video-import.service.ts View File

@@ -0,0 +1,56 @@
import { catchError } from 'rxjs/operators'
import { HttpClient } from '@angular/common/http'
import { Injectable } from '@angular/core'
import { Observable } from 'rxjs'
import { VideoImport } from '../../../../../shared'
import { environment } from '../../../environments/environment'
import { RestExtractor, RestService } from '../rest'
import { VideoImportCreate } from '../../../../../shared/models/videos/video-import-create.model'
import { objectToFormData } from '@app/shared/misc/utils'
import { VideoUpdate } from '../../../../../shared/models/videos'

@Injectable()
export class VideoImportService {
private static BASE_VIDEO_IMPORT_URL = environment.apiUrl + '/api/v1/videos/imports/'

constructor (
private authHttp: HttpClient,
private restService: RestService,
private restExtractor: RestExtractor
) {}

importVideo (targetUrl: string, video: VideoUpdate): Observable<VideoImport> {
const url = VideoImportService.BASE_VIDEO_IMPORT_URL
const language = video.language || null
const licence = video.licence || null
const category = video.category || null
const description = video.description || null
const support = video.support || null
const scheduleUpdate = video.scheduleUpdate || null

const body: VideoImportCreate = {
targetUrl,

name: video.name,
category,
licence,
language,
support,
description,
channelId: video.channelId,
privacy: video.privacy,
tags: video.tags,
nsfw: video.nsfw,
waitTranscoding: video.waitTranscoding,
commentsEnabled: video.commentsEnabled,
thumbnailfile: video.thumbnailfile,
previewfile: video.previewfile,
scheduleUpdate
}

const data = objectToFormData(body)
return this.authHttp.post<VideoImport>(url, data)
.pipe(catchError(res => this.restExtractor.handleError(res)))
}

}

+ 20
- 20
client/src/app/shared/video/video-edit.model.ts View File

@@ -1,7 +1,7 @@
import { VideoDetails } from './video-details.model'
import { VideoPrivacy } from '../../../../../shared/models/videos/video-privacy.enum'
import { VideoUpdate } from '../../../../../shared/models/videos'
import { VideoScheduleUpdate } from '../../../../../shared/models/videos/video-schedule-update.model'
import { Video } from '../../../../../shared/models/videos/video.model'

export class VideoEdit implements VideoUpdate {
static readonly SPECIAL_SCHEDULED_PRIVACY = -1
@@ -26,26 +26,26 @@ export class VideoEdit implements VideoUpdate {
id?: number
scheduleUpdate?: VideoScheduleUpdate

constructor (videoDetails?: VideoDetails) {
if (videoDetails) {
this.id = videoDetails.id
this.uuid = videoDetails.uuid
this.category = videoDetails.category.id
this.licence = videoDetails.licence.id
this.language = videoDetails.language.id
this.description = videoDetails.description
this.name = videoDetails.name
this.tags = videoDetails.tags
this.nsfw = videoDetails.nsfw
this.commentsEnabled = videoDetails.commentsEnabled
this.waitTranscoding = videoDetails.waitTranscoding
this.channelId = videoDetails.channel.id
this.privacy = videoDetails.privacy.id
this.support = videoDetails.support
this.thumbnailUrl = videoDetails.thumbnailUrl
this.previewUrl = videoDetails.previewUrl
constructor (video?: Video & { tags: string[], commentsEnabled: boolean, support: string, thumbnailUrl: string, previewUrl: string }) {
if (video) {
this.id = video.id
this.uuid = video.uuid
this.category = video.category.id
this.licence = video.licence.id
this.language = video.language.id
this.description = video.description
this.name = video.name
this.tags = video.tags
this.nsfw = video.nsfw
this.commentsEnabled = video.commentsEnabled
this.waitTranscoding = video.waitTranscoding
this.channelId = video.channel.id
this.privacy = video.privacy.id
this.support = video.support
this.thumbnailUrl = video.thumbnailUrl
this.previewUrl = video.previewUrl

this.scheduleUpdate = videoDetails.scheduledUpdate
this.scheduleUpdate = video.scheduledUpdate
}
}



+ 10
- 58
client/src/app/videos/+video-edit/video-add.component.html View File

@@ -1,65 +1,17 @@
<div class="margin-content">
<div class="title-page title-page-single">
<ng-container *ngIf="!videoFileName" i18n>Upload your video</ng-container>
<ng-container *ngIf="videoFileName" i18n>Upload {{ videoFileName }}</ng-container>
<ng-container *ngIf="secondStepType === 'import'" i18n>Import {{ videoName }}</ng-container>
<ng-container *ngIf="secondStepType === 'upload'" i18n>Upload {{ videoName }}</ng-container>
</div>

<div *ngIf="!isUploadingVideo" class="upload-video-container">
<div class="upload-video">
<div class="icon icon-upload"></div>
<tabset class="video-add-tabset root-tabset bootstrap" [ngClass]="{ 'hide-nav': secondStepType !== undefined }">

<div class="button-file">
<span i18n>Select the file to upload</span>
<input #videofileInput type="file" name="videofile" id="videofile" [accept]="videoExtensions" (change)="fileChange()" />
</div>
<span class="button-file-extension">(.mp4, .webm, .ogv)</span>
<tab i18n-heading heading="Upload your video">
<my-video-upload #videoUpload (firstStepDone)="onFirstStepDone('upload', $event)"></my-video-upload>
</tab>

<div class="form-group form-group-channel">
<label i18n for="first-step-channel">Channel</label>
<div class="peertube-select-container">
<select id="first-step-channel" [(ngModel)]="firstStepChannelId">
<option *ngFor="let channel of userVideoChannels" [value]="channel.id">{{ channel.label }}</option>
</select>
</div>
</div>

<div class="form-group">
<label i18n for="first-step-privacy">Privacy</label>
<div class="peertube-select-container">
<select id="first-step-privacy" [(ngModel)]="firstStepPrivacyId">
<option *ngFor="let privacy of videoPrivacies" [value]="privacy.id">{{ privacy.label }}</option>
<option [value]="SPECIAL_SCHEDULED_PRIVACY">Scheduled</option>
</select>
</div>
</div>
</div>
</div>

<div *ngIf="isUploadingVideo" class="upload-progress-cancel">
<p-progressBar
[value]="videoUploadPercents"
[ngClass]="{ processing: videoUploadPercents === 100 && videoUploaded === false }"
></p-progressBar>
<input *ngIf="videoUploaded === false" type="button" value="Cancel" (click)="cancelUpload()" />
</div>

<!-- Hidden because we want to load the component -->
<form [hidden]="!isUploadingVideo" novalidate [formGroup]="form">
<my-video-edit
[form]="form" [formErrors]="formErrors" [videoCaptions]="videoCaptions"
[validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels"
></my-video-edit>

<div class="submit-container">
<div i18n *ngIf="videoUploaded === false" class="message-submit">Publish will be available when upload is finished</div>

<div class="submit-button"
(click)="updateSecondStep()"
[ngClass]="{ disabled: !form.valid || isUpdatingVideo === true || videoUploaded !== true }"
>
<span class="icon icon-validate"></span>
<input type="button" i18n-value value="Publish" />
</div>
</div>
</form>
<tab i18n-heading heading="Import your video">
<my-video-import #videoImport (firstStepDone)="onFirstStepDone('import', $event)"></my-video-import>
</tab>
</tabset>
</div>

+ 35
- 82
client/src/app/videos/+video-edit/video-add.component.scss View File

@@ -1,101 +1,54 @@
@import '_variables';
@import '_mixins';

.upload-video-container {
border-radius: 3px;
background-color: #F7F7F7;
border: 3px solid #EAEAEA;
width: 100%;
height: 440px;
margin-top: 40px;
display: flex;
justify-content: center;
align-items: center;
$border-width: 3px;
$border-type: solid;
$border-color: #EAEAEA;

.peertube-select-container {
@include peertube-select-container(190px);
}

.upload-video {
display: flex;
flex-direction: column;
align-items: center;

.form-group-channel {
margin-bottom: 20px;
}

.icon.icon-upload {
@include icon(90px);
margin-bottom: 25px;
cursor: default;

background-image: url('../../../assets/images/video/upload.svg');
}

.button-file {
@include peertube-button-file(auto);

min-width: 190px;
}
$background-color: #F7F7F7;

.button-file-extension {
display: block;
font-size: 12px;
margin-top: 5px;
}
}

.form-group-channel {
margin-top: 35px;
/deep/ tabset.root-tabset.video-add-tabset {
&.hide-nav .nav {
display: none !important;
}
}

.upload-progress-cancel {
display: flex;
margin-top: 25px;
margin-bottom: 40px;
& > .nav {

p-progressBar {
flex-grow: 1;

/deep/ .ui-progressbar {
font-size: 15px !important;
color: #fff !important;
height: 30px !important;
line-height: 30px !important;
border-radius: 3px !important;
background-color: rgba(11, 204, 41, 0.16) !important;

.ui-progressbar-value {
background-color: #0BCC29 !important;
}
border-bottom: $border-width $border-type $border-color;
margin: 0 !important;

.ui-progressbar-label {
text-align: left;
padding-left: 18px;
margin-top: 0 !important;
}
& > li {
margin-bottom: -$border-width;
}

&.processing {
/deep/ .ui-progressbar-label {
// Same color as background to hide "100%"
color: rgba(11, 204, 41, 0.16) !important;
.nav-link {
height: 40px !important;
padding: 0 30px !important;
font-size: 15px;

&.active {
border: $border-width $border-type $border-color;
border-bottom: none;
background-color: $background-color !important;

&::before {
content: 'Processing...';
color: #fff;
span {
border-bottom: 2px solid #F1680D;
font-weight: $font-bold;
}
}
}
}

input {
@include peertube-button;
@include grey-button;
.upload-video-container {
border: $border-width $border-type $border-color;
border-top: none;

margin-left: 10px;
background-color: $background-color;
border-radius: 3px;
width: 100%;
height: 440px;
display: flex;
justify-content: center;
align-items: center;
}
}

}

+ 15
- 237
client/src/app/videos/+video-edit/video-add.component.ts View File

@@ -1,251 +1,29 @@
import { HttpEventType, HttpResponse } from '@angular/common/http'
import { Component, OnDestroy, OnInit, ViewChild } from '@angular/core'
import { Router } from '@angular/router'
import { UserService } from '@app/shared'
import { Component, ViewChild } from '@angular/core'
import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.service'
import { LoadingBarService } from '@ngx-loading-bar/core'
import { NotificationsService } from 'angular2-notifications'
import { BytesPipe } from 'ngx-pipes'
import { Subscription } from 'rxjs'
import { VideoConstant, VideoPrivacy } from '../../../../../shared/models/videos'
import { AuthService, ServerService } from '../../core'
import { FormReactive } from '../../shared'
import { populateAsyncUserVideoChannels } from '../../shared/misc/utils'
import { VideoEdit } from '../../shared/video/video-edit.model'
import { VideoService } from '../../shared/video/video.service'
import { I18n } from '@ngx-translate/i18n-polyfill'
import { FormValidatorService } from '@app/shared/forms/form-validators/form-validator.service'
import { switchMap } from 'rxjs/operators'
import { VideoCaptionService } from '@app/shared/video-caption'
import { VideoCaptionEdit } from '@app/shared/video-caption/video-caption-edit.model'
import { VideoImportComponent } from '@app/videos/+video-edit/video-import.component'
import { VideoUploadComponent } from '@app/videos/+video-edit/video-upload.component'

@Component({
selector: 'my-videos-add',
templateUrl: './video-add.component.html',
styleUrls: [
'./shared/video-edit.component.scss',
'./video-add.component.scss'
]
styleUrls: [ './video-add.component.scss' ]
})
export class VideoAddComponent extends FormReactive implements OnInit, OnDestroy, CanComponentDeactivate {
@ViewChild('videofileInput') videofileInput
export class VideoAddComponent implements CanComponentDeactivate {
@ViewChild('videoUpload') videoUpload: VideoUploadComponent
@ViewChild('videoImport') videoImport: VideoImportComponent

// So that it can be accessed in the template
readonly SPECIAL_SCHEDULED_PRIVACY = VideoEdit.SPECIAL_SCHEDULED_PRIVACY
secondStepType: 'upload' | 'import'
videoName: string

isUploadingVideo = false
isUpdatingVideo = false
videoUploaded = false
videoUploadObservable: Subscription = null
videoUploadPercents = 0
videoUploadedIds = {
id: 0,
uuid: ''
}
videoFileName: string

userVideoChannels: { id: number, label: string, support: string }[] = []
userVideoQuotaUsed = 0
videoPrivacies: VideoConstant<string>[] = []
firstStepPrivacyId = 0
firstStepChannelId = 0
videoCaptions: VideoCaptionEdit[] = []

constructor (
protected formValidatorService: FormValidatorService,
private router: Router,
private notificationsService: NotificationsService,
private authService: AuthService,
private userService: UserService,
private serverService: ServerService,
private videoService: VideoService,
private loadingBar: LoadingBarService,
private i18n: I18n,
private videoCaptionService: VideoCaptionService
) {
super()
}

get videoExtensions () {
return this.serverService.getConfig().video.file.extensions.join(',')
}

ngOnInit () {
this.buildForm({})

populateAsyncUserVideoChannels(this.authService, this.userVideoChannels)
.then(() => this.firstStepChannelId = this.userVideoChannels[0].id)

this.userService.getMyVideoQuotaUsed()
.subscribe(data => this.userVideoQuotaUsed = data.videoQuotaUsed)

this.serverService.videoPrivaciesLoaded
.subscribe(
() => {
this.videoPrivacies = this.serverService.getVideoPrivacies()

// Public by default
this.firstStepPrivacyId = VideoPrivacy.PUBLIC
})
}

ngOnDestroy () {
if (this.videoUploadObservable) {
this.videoUploadObservable.unsubscribe()
}
onFirstStepDone (type: 'upload' | 'import', videoName: string) {
this.secondStepType = type
this.videoName = videoName
}

canDeactivate () {
let text = ''

if (this.videoUploaded === true) {
// FIXME: cannot concatenate strings inside i18n service :/
text = this.i18n('Your video was uploaded in your account and is private.') +
this.i18n('But associated data (tags, description...) will be lost, are you sure you want to leave this page?')
} else {
text = this.i18n('Your video is not uploaded yet, are you sure you want to leave this page?')
}

return {
canDeactivate: !this.isUploadingVideo,
text
}
}

fileChange () {
this.uploadFirstStep()
}

checkForm () {
this.forceCheck()

return this.form.valid
}

cancelUpload () {
if (this.videoUploadObservable !== null) {
this.videoUploadObservable.unsubscribe()
this.isUploadingVideo = false
this.videoUploadPercents = 0
this.videoUploadObservable = null
this.notificationsService.info(this.i18n('Info'), this.i18n('Upload cancelled'))
}
}

uploadFirstStep () {
const videofile = this.videofileInput.nativeElement.files[0] as File
if (!videofile) return

// Cannot upload videos > 8GB for now
if (videofile.size > 8 * 1024 * 1024 * 1024) {
this.notificationsService.error(this.i18n('Error'), this.i18n('We are sorry but PeerTube cannot handle videos > 8GB'))
return
}

const videoQuota = this.authService.getUser().videoQuota
if (videoQuota !== -1 && (this.userVideoQuotaUsed + videofile.size) > videoQuota) {
const bytePipes = new BytesPipe()

const msg = this.i18n(
'Your video quota is exceeded with this video (video size: {{ videoSize }}, used: {{ videoQuotaUsed }}, quota: {{ videoQuota }})',
{
videoSize: bytePipes.transform(videofile.size, 0),
videoQuotaUsed: bytePipes.transform(this.userVideoQuotaUsed, 0),
videoQuota: bytePipes.transform(videoQuota, 0)
}
)
this.notificationsService.error(this.i18n('Error'), msg)
return
}

this.videoFileName = videofile.name

const nameWithoutExtension = videofile.name.replace(/\.[^/.]+$/, '')
let name: string

// If the name of the file is very small, keep the extension
if (nameWithoutExtension.length < 3) name = videofile.name
else name = nameWithoutExtension

const privacy = this.firstStepPrivacyId.toString()
const nsfw = false
const waitTranscoding = true
const commentsEnabled = true
const channelId = this.firstStepChannelId.toString()

const formData = new FormData()
formData.append('name', name)
// Put the video "private" -> we are waiting the user validation of the second step
formData.append('privacy', VideoPrivacy.PRIVATE.toString())
formData.append('nsfw', '' + nsfw)
formData.append('commentsEnabled', '' + commentsEnabled)
formData.append('waitTranscoding', '' + waitTranscoding)
formData.append('channelId', '' + channelId)
formData.append('videofile', videofile)

this.isUploadingVideo = true
this.form.patchValue({
name,
privacy,
nsfw,
channelId
})

this.videoUploadObservable = this.videoService.uploadVideo(formData).subscribe(
event => {
if (event.type === HttpEventType.UploadProgress) {
this.videoUploadPercents = Math.round(100 * event.loaded / event.total)
} else if (event instanceof HttpResponse) {
this.videoUploaded = true

this.videoUploadedIds = event.body.video

this.videoUploadObservable = null
}
},

err => {
// Reset progress
this.isUploadingVideo = false
this.videoUploadPercents = 0
this.videoUploadObservable = null
this.notificationsService.error(this.i18n('Error'), err.message)
}
)
}

updateSecondStep () {
if (this.checkForm() === false) {
return
}

const video = new VideoEdit()
video.patch(this.form.value)
video.id = this.videoUploadedIds.id
video.uuid = this.videoUploadedIds.uuid

this.isUpdatingVideo = true
this.loadingBar.start()
this.videoService.updateVideo(video)
.pipe(
// Then update captions
switchMap(() => this.videoCaptionService.updateCaptions(video.id, this.videoCaptions))
)
.subscribe(
() => {
this.isUpdatingVideo = false
this.isUploadingVideo = false
this.loadingBar.complete()

this.notificationsService.success(this.i18n('Success'), this.i18n('Video published.'))
this.router.navigate([ '/videos/watch', video.uuid ])
},
if (this.secondStepType === 'upload') return this.videoUpload.canDeactivate()
if (this.secondStepType === 'import') return this.videoImport.canDeactivate()

err => {
this.isUpdatingVideo = false
this.notificationsService.error(this.i18n('Error'), err.message)
console.error(err)
}
)
return { canDeactivate: true }
}
}

+ 5
- 1
client/src/app/videos/+video-edit/video-add.module.ts View File

@@ -5,6 +5,8 @@ import { VideoEditModule } from './shared/video-edit.module'
import { VideoAddRoutingModule } from './video-add-routing.module'
import { VideoAddComponent } from './video-add.component'
import { CanDeactivateGuard } from '../../shared/guards/can-deactivate-guard.service'
import { VideoUploadComponent } from '@app/videos/+video-edit/video-upload.component'
import { VideoImportComponent } from '@app/videos/+video-edit/video-import.component'

@NgModule({
imports: [
@@ -14,7 +16,9 @@ import { CanDeactivateGuard } from '../../shared/guards/can-deactivate-guard.ser
ProgressBarModule
],
declarations: [
VideoAddComponent
VideoAddComponent,
VideoUploadComponent,
VideoImportComponent
],
exports: [
VideoAddComponent


+ 55
- 0
client/src/app/videos/+video-edit/video-import.component.html View File

@@ -0,0 +1,55 @@
<div *ngIf="!hasImportedVideo" class="upload-video-container">
<div class="import-video">
<div class="icon icon-upload"></div>

<div class="form-group">
<label i18n for="targetUrl">URL</label>
<input type="text" id="targetUrl" [(ngModel)]="targetUrl" />
</div>

<div class="form-group">
<label i18n for="first-step-channel">Channel</label>
<div class="peertube-select-container">
<select id="first-step-channel" [(ngModel)]="firstStepChannelId">
<option *ngFor="let channel of userVideoChannels" [value]="channel.id">{{ channel.label }}</option>
</select>
</div>
</div>

<div class="form-group">
<label i18n for="first-step-privacy">Privacy</label>
<div class="peertube-select-container">
<select id="first-step-privacy" [(ngModel)]="firstStepPrivacyId">
<option *ngFor="let privacy of videoPrivacies" [value]="privacy.id">{{ privacy.label }}</option>
</select>
</div>
</div>

<input
type="button" i18n-value value="Import"
[disabled]="!isTargetUrlValid() || isImportingVideo" (click)="importVideo()"
/>
</div>
</div>

<div *ngIf="hasImportedVideo" class="alert alert-info" i18n>
Congratulations, the video behind {{ targetUrl }} will be imported! You can already add information about this video.
</div>

<!-- Hidden because we want to load the component -->
<form [hidden]="!hasImportedVideo" novalidate [formGroup]="form">
<my-video-edit
[form]="form" [formErrors]="formErrors" [videoCaptions]="videoCaptions" [schedulePublicationPossible]="false"
[validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels"
></my-video-edit>

<div class="submit-container">
<div class="submit-button"
(click)="updateSecondStep()"
[ngClass]="{ disabled: !form.valid || isUpdatingVideo === true }"
>
<span class="icon icon-validate"></span>
<input type="button" i18n-value value="Update" />
</div>
</div>
</form>

+ 37
- 0
client/src/app/videos/+video-edit/video-import.component.scss View File

@@ -0,0 +1,37 @@
@import '_variables';
@import '_mixins';

$width-size: 190px;

.peertube-select-container {
@include peertube-select-container($width-size);
}

.import-video {
display: flex;
flex-direction: column;
align-items: center;

.icon.icon-upload {
@include icon(90px);
margin-bottom: 25px;
cursor: default;

background-image: url('../../../assets/images/video/upload.svg');
}

input[type=text] {
@include peertube-input-text($width-size);
display: block;
}

input[type=button] {
@include peertube-button;
@include orange-button;

width: $width-size;
margin-top: 30px;
}
}



+ 161
- 0
client/src/app/videos/+video-edit/video-import.component.ts View File

@@ -0,0 +1,161 @@
import { Component, EventEmitter, OnInit, Output } from '@angular/core'
import { Router } from '@angular/router'
import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.service'
import { NotificationsService } from 'angular2-notifications'
import { VideoConstant, VideoPrivacy, VideoUpdate } from '../../../../../shared/models/videos'
import { AuthService, ServerService } from '../../core'
import { FormReactive } from '../../shared'
import { populateAsyncUserVideoChannels } from '../../shared/misc/utils'
import { VideoService } from '../../shared/video/video.service'
import { I18n } from '@ngx-translate/i18n-polyfill'
import { FormValidatorService } from '@app/shared/forms/form-validators/form-validator.service'
import { VideoCaptionEdit } from '@app/shared/video-caption/video-caption-edit.model'
import { VideoImportService } from '@app/shared/video-import'
import { VideoEdit } from '@app/shared/video/video-edit.model'
import { switchMap } from 'rxjs/operators'
import { LoadingBarService } from '@ngx-loading-bar/core'
import { VideoCaptionService } from '@app/shared/video-caption'

@Component({
selector: 'my-video-import',
templateUrl: './video-import.component.html',
styleUrls: [
'./shared/video-edit.component.scss',
'./video-import.component.scss'
]
})
export class VideoImportComponent extends FormReactive implements OnInit, CanComponentDeactivate {
@Output() firstStepDone = new EventEmitter<string>()

targetUrl = ''
videoFileName: string

isImportingVideo = false
hasImportedVideo = false
isUpdatingVideo = false

userVideoChannels: { id: number, label: string, support: string }[] = []
videoPrivacies: VideoConstant<string>[] = []
videoCaptions: VideoCaptionEdit[] = []

firstStepPrivacyId = 0
firstStepChannelId = 0
video: VideoEdit

constructor (
protected formValidatorService: FormValidatorService,
private router: Router,
private loadingBar: LoadingBarService,
private notificationsService: NotificationsService,
private authService: AuthService,
private serverService: ServerService,
private videoService: VideoService,
private videoImportService: VideoImportService,
private videoCaptionService: VideoCaptionService,
private i18n: I18n
) {
super()
}

ngOnInit () {
this.buildForm({})

populateAsyncUserVideoChannels(this.authService, this.userVideoChannels)
.then(() => this.firstStepChannelId = this.userVideoChannels[ 0 ].id)

this.serverService.videoPrivaciesLoaded
.subscribe(
() => {
this.videoPrivacies = this.serverService.getVideoPrivacies()

// Private by default
this.firstStepPrivacyId = VideoPrivacy.PRIVATE
})
}

canDeactivate () {
return { canDeactivate: true }
}

checkForm () {
this.forceCheck()

return this.form.valid
}

isTargetUrlValid () {
return this.targetUrl && this.targetUrl.match(/https?:\/\//)
}

importVideo () {
this.isImportingVideo = true

const videoUpdate: VideoUpdate = {
privacy: this.firstStepPrivacyId,
waitTranscoding: false,
commentsEnabled: true,
channelId: this.firstStepChannelId
}

this.videoImportService.importVideo(this.targetUrl, videoUpdate).subscribe(
res => {
this.firstStepDone.emit(res.video.name)
this.isImportingVideo = false
this.hasImportedVideo = true

this.video = new VideoEdit(Object.assign(res.video, {
commentsEnabled: videoUpdate.commentsEnabled,
support: null,
thumbnailUrl: null,
previewUrl: null
}))
this.hydrateFormFromVideo()
},

err => {
this.isImportingVideo = false
this.notificationsService.error(this.i18n('Error'), err.message)
}
)
}

updateSecondStep () {
if (this.checkForm() === false) {
return
}

this.video.patch(this.form.value)

this.loadingBar.start()
this.isUpdatingVideo = true

// Update the video
this.videoService.updateVideo(this.video)
.pipe(
// Then update captions
switchMap(() => this.videoCaptionService.updateCaptions(this.video.id, this.videoCaptions))
)
.subscribe(
() => {
this.isUpdatingVideo = false
this.loadingBar.complete()
this.notificationsService.success(this.i18n('Success'), this.i18n('Video to import updated.'))

// TODO: route to imports list
// this.router.navigate([ '/videos/watch', this.video.uuid ])
},

err => {
this.loadingBar.complete()
this.isUpdatingVideo = false
this.notificationsService.error(this.i18n('Error'), err.message)
console.error(err)
}
)

}

private hydrateFormFromVideo () {
this.form.patchValue(this.video.toFormPatch())
}
}

+ 0
- 1
client/src/app/videos/+video-edit/video-update.component.ts View File

@@ -126,7 +126,6 @@ export class VideoUpdateComponent extends FormReactive implements OnInit {
console.error(err)
}
)

}

private hydrateFormFromVideo () {


+ 58
- 0
client/src/app/videos/+video-edit/video-upload.component.html View File

@@ -0,0 +1,58 @@
<div *ngIf="!isUploadingVideo" class="upload-video-container">
<div class="upload-video">
<div class="icon icon-upload"></div>

<div class="button-file">
<span i18n>Select the file to upload</span>
<input #videofileInput type="file" name="videofile" id="videofile" [accept]="videoExtensions" (change)="fileChange()" />
</div>
<span class="button-file-extension">(.mp4, .webm, .ogv)</span>

<div class="form-group form-group-channel">
<label i18n for="first-step-channel">Channel</label>
<div class="peertube-select-container">
<select id="first-step-channel" [(ngModel)]="firstStepChannelId">
<option *ngFor="let channel of userVideoChannels" [value]="channel.id">{{ channel.label }}</option>
</select>
</div>
</div>

<div class="form-group">
<label i18n for="first-step-privacy">Privacy</label>
<div class="peertube-select-container">
<select id="first-step-privacy" [(ngModel)]="firstStepPrivacyId">
<option *ngFor="let privacy of videoPrivacies" [value]="privacy.id">{{ privacy.label }}</option>
<option [value]="SPECIAL_SCHEDULED_PRIVACY">Scheduled</option>
</select>
</div>
</div>
</div>
</div>

<div *ngIf="isUploadingVideo" class="upload-progress-cancel">
<p-progressBar
[value]="videoUploadPercents"
[ngClass]="{ processing: videoUploadPercents === 100 && videoUploaded === false }"
></p-progressBar>
<input *ngIf="videoUploaded === false" type="button" value="Cancel" (click)="cancelUpload()" />
</div>

<!-- Hidden because we want to load the component -->
<form [hidden]="!isUploadingVideo" novalidate [formGroup]="form">
<my-video-edit
[form]="form" [formErrors]="formErrors" [videoCaptions]="videoCaptions"
[validationMessages]="validationMessages" [videoPrivacies]="videoPrivacies" [userVideoChannels]="userVideoChannels"
></my-video-edit>

<div class="submit-container">
<div i18n *ngIf="videoUploaded === false" class="message-submit">Publish will be available when upload is finished</div>

<div class="submit-button"
(click)="updateSecondStep()"
[ngClass]="{ disabled: !form.valid || isUpdatingVideo === true || videoUploaded !== true }"
>
<span class="icon icon-validate"></span>
<input type="button" i18n-value value="Publish" />
</div>
</div>
</form>

+ 85
- 0
client/src/app/videos/+video-edit/video-upload.component.scss View File

@@ -0,0 +1,85 @@
@import '_variables';
@import '_mixins';

.peertube-select-container {
@include peertube-select-container(190px);
}

.upload-video {
display: flex;
flex-direction: column;
align-items: center;

.form-group-channel {
margin-bottom: 20px;
margin-top: 35px;
}

.icon.icon-upload {
@include icon(90px);
margin-bottom: 25px;
cursor: default;

background-image: url('../../../assets/images/video/upload.svg');
}

.button-file {
@include peertube-button-file(auto);

min-width: 190px;
}

.button-file-extension {
display: block;
font-size: 12px;
margin-top: 5px;
}
}

.upload-progress-cancel {
display: flex;
margin-top: 25px;
margin-bottom: 40px;

p-progressBar {
flex-grow: 1;

/deep/ .ui-progressbar {
font-size: 15px !important;
color: #fff !important;
height: 30px !important;
line-height: 30px !important;
border-radius: 3px !important;
background-color: rgba(11, 204, 41, 0.16) !important;

.ui-progressbar-value {
background-color: #0BCC29 !important;
}

.ui-progressbar-label {
text-align: left;
padding-left: 18px;
margin-top: 0 !important;
}
}

&.processing {
/deep/ .ui-progressbar-label {
// Same color as background to hide "100%"
color: rgba(11, 204, 41, 0.16) !important;

&::before {
content: 'Processing...';
color: #fff;
}
}
}
}

input {
@include peertube-button;
@include grey-button;

margin-left: 10px;
}
}

+ 251
- 0
client/src/app/videos/+video-edit/video-upload.component.ts View File

@@ -0,0 +1,251 @@
import { HttpEventType, HttpResponse } from '@angular/common/http'
import { Component, EventEmitter, OnDestroy, OnInit, Output, ViewChild } from '@angular/core'
import { Router } from '@angular/router'
import { UserService } from '@app/shared'
import { CanComponentDeactivate } from '@app/shared/guards/can-deactivate-guard.service'
import { LoadingBarService } from '@ngx-loading-bar/core'
import { NotificationsService } from 'angular2-notifications'
import { BytesPipe } from 'ngx-pipes'
import { Subscription } from 'rxjs'
import { VideoConstant, VideoPrivacy } from '../../../../../shared/models/videos'
import { AuthService, ServerService } from '../../core'
import { FormReactive } from '../../shared'
import { populateAsyncUserVideoChannels } from '../../shared/misc/utils'
import { VideoEdit } from '../../shared/video/video-edit.model'
import { VideoService } from '../../shared/video/video.service'
import { I18n } from '@ngx-translate/i18n-polyfill'
import { FormValidatorService } from '@app/shared/forms/form-validators/form-validator.service'
import { switchMap } from 'rxjs/operators'
import { VideoCaptionService } from '@app/shared/video-caption'
import { VideoCaptionEdit } from '@app/shared/video-caption/video-caption-edit.model'

@Component({
selector: 'my-video-upload',
templateUrl: './video-upload.component.html',
styleUrls: [
'./shared/video-edit.component.scss',
'./video-upload.component.scss'
]
})
export class VideoUploadComponent extends FormReactive implements OnInit, OnDestroy, CanComponentDeactivate {
@Output() firstStepDone = new EventEmitter<string>()
@ViewChild('videofileInput') videofileInput

// So that it can be accessed in the template
readonly SPECIAL_SCHEDULED_PRIVACY = VideoEdit.SPECIAL_SCHEDULED_PRIVACY

isUploadingVideo = false
isUpdatingVideo = false
videoUploaded = false
videoUploadObservable: Subscription = null
videoUploadPercents = 0
videoUploadedIds = {
id: 0,
uuid: ''
}

userVideoChannels: { id: number, label: string, support: string }[] = []
userVideoQuotaUsed = 0
videoPrivacies: VideoConstant<string>[] = []
firstStepPrivacyId = 0
firstStepChannelId = 0
videoCaptions: VideoCaptionEdit[] = []

constructor (
protected formValidatorService: FormValidatorService,
private router: Router,
private notificationsService: NotificationsService,
private authService: AuthService,
private userService: UserService,
private serverService: ServerService,
private videoService: VideoService,
private loadingBar: LoadingBarService,
private i18n: I18n,
private videoCaptionService: VideoCaptionService
) {
super()
}

get videoExtensions () {
return this.serverService.getConfig().video.file.extensions.join(',')
}

ngOnInit () {
this.buildForm({})

populateAsyncUserVideoChannels(this.authService, this.userVideoChannels)
.then(() => this.firstStepChannelId = this.userVideoChannels[0].id)

this.userService.getMyVideoQuotaUsed()
.subscribe(data => this.userVideoQuotaUsed = data.videoQuotaUsed)

this.serverService.videoPrivaciesLoaded
.subscribe(
() => {
this.videoPrivacies = this.serverService.getVideoPrivacies()

// Public by default
this.firstStepPrivacyId = VideoPrivacy.PUBLIC
})
}

ngOnDestroy () {
if (this.videoUploadObservable) {
this.videoUploadObservable.unsubscribe()
}
}

canDeactivate () {
let text = ''

if (this.videoUploaded === true) {
// FIXME: cannot concatenate strings inside i18n service :/
text = this.i18n('Your video was uploaded in your account and is private.') +
this.i18n('But associated data (tags, description...) will be lost, are you sure you want to leave this page?')
} else {
text = this.i18n('Your video is not uploaded yet, are you sure you want to leave this page?')
}

return {
canDeactivate: !this.isUploadingVideo,
text
}
}

fileChange () {
this.uploadFirstStep()
}

checkForm () {
this.forceCheck()

return this.form.valid
}

cancelUpload () {
if (this.videoUploadObservable !== null) {
this.videoUploadObservable.unsubscribe()
this.isUploadingVideo = false
this.videoUploadPercents = 0
this.videoUploadObservable = null
this.notificationsService.info(this.i18n('Info'), this.i18n('Upload cancelled'))
}
}

uploadFirstStep () {
const videofile = this.videofileInput.nativeElement.files[0] as File
if (!videofile) return

// Cannot upload videos > 8GB for now
if (videofile.size > 8 * 1024 * 1024 * 1024) {
this.notificationsService.error(this.i18n('Error'), this.i18n('We are sorry but PeerTube cannot handle videos > 8GB'))
return
}

const videoQuota = this.authService.getUser().videoQuota
if (videoQuota !== -1 && (this.userVideoQuotaUsed + videofile.size) > videoQuota) {
const bytePipes = new BytesPipe()

const msg = this.i18n(
'Your video quota is exceeded with this video (video size: {{ videoSize }}, used: {{ videoQuotaUsed }}, quota: {{ videoQuota }})',
{
videoSize: bytePipes.transform(videofile.size, 0),
videoQuotaUsed: bytePipes.transform(this.userVideoQuotaUsed, 0),
videoQuota: bytePipes.transform(videoQuota, 0)
}
)
this.notificationsService.error(this.i18n('Error'), msg)
return
}

const nameWithoutExtension = videofile.name.replace(/\.[^/.]+$/, '')
let name: string

// If the name of the file is very small, keep the extension
if (nameWithoutExtension.length < 3) name = videofile.name
else name = nameWithoutExtension

const privacy = this.firstStepPrivacyId.toString()
const nsfw = false
const waitTranscoding = true
const commentsEnabled = true
const channelId = this.firstStepChannelId.toString()

const formData = new FormData()
formData.append('name', name)
// Put the video "private" -> we are waiting the user validation of the second step
formData.append('privacy', VideoPrivacy.PRIVATE.toString())
formData.append('nsfw', '' + nsfw)
formData.append('commentsEnabled', '' + commentsEnabled)
formData.append('waitTranscoding', '' + waitTranscoding)
formData.append('channelId', '' + channelId)
formData.append('videofile', videofile)

this.isUploadingVideo = true
this.firstStepDone.emit(name)

this.form.patchValue({
name,
privacy,
nsfw,
channelId
})

this.videoUploadObservable = this.videoService.uploadVideo(formData).subscribe(
event => {
if (event.type === HttpEventType.UploadProgress) {
this.videoUploadPercents = Math.round(100 * event.loaded / event.total)
} else if (event instanceof HttpResponse) {
this.videoUploaded = true

this.videoUploadedIds = event.body.video

this.videoUploadObservable = null
}
},

err => {
// Reset progress
this.isUploadingVideo = false
this.videoUploadPercents = 0
this.videoUploadObservable = null
this.notificationsService.error(this.i18n('Error'), err.message)
}
)
}

updateSecondStep () {
if (this.checkForm() === false) {
return
}

const video = new VideoEdit()
video.patch(this.form.value)
video.id = this.videoUploadedIds.id
video.uuid = this.videoUploadedIds.uuid

this.isUpdatingVideo = true
this.loadingBar.start()
this.videoService.updateVideo(video)
.pipe(
// Then update captions
switchMap(() => this.videoCaptionService.updateCaptions(video.id, this.videoCaptions))
)
.subscribe(
() => {
this.isUpdatingVideo = false
this.isUploadingVideo = false
this.loadingBar.complete()

this.notificationsService.success(this.i18n('Success'), this.i18n('Video published.'))
this.router.navigate([ '/videos/watch', video.uuid ])
},

err => {
this.isUpdatingVideo = false
this.notificationsService.error(this.i18n('Error'), err.message)
console.error(err)
}
)
}
}

+ 6
- 0
config/default.yaml View File

@@ -92,6 +92,12 @@ transcoding:
720p: false
1080p: false

import:
# Add ability for your users to import remote videos (from YouTube, torrent...)
videos:
http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
enabled: true

instance:
name: 'PeerTube'
short_description: 'PeerTube, a federated (ActivityPub) video streaming platform using P2P (BitTorrent) directly in the web browser with WebTorrent and Angular.'


+ 4
- 3
package.json View File

@@ -16,6 +16,7 @@
"type": "git",
"url": "git://github.com/Chocobozzz/PeerTube.git"
},
"typings": "*.d.ts",
"scripts": {
"e2e": "scripty",
"build": "SCRIPTY_PARALLEL=true scripty",
@@ -132,7 +133,8 @@
"validator": "^10.2.0",
"webfinger.js": "^2.6.6",
"winston": "3.0.0",
"ws": "^5.0.0"
"ws": "^5.0.0",
"youtube-dl": "^1.12.2"
},
"devDependencies": {
"@types/async": "^2.0.40",
@@ -184,8 +186,7 @@
"tslint-config-standard": "^7.0.0",
"typescript": "^2.5.2",
"webtorrent": "^0.100.0",
"xliff": "^3.0.1",
"youtube-dl": "^1.12.2"
"xliff": "^3.0.1"
},
"scripty": {
"silent": true


+ 151
- 0
server/controllers/api/videos/import.ts View File

@@ -0,0 +1,151 @@
import * as express from 'express'
import { auditLoggerFactory } from '../../../helpers/audit-logger'
import {
asyncMiddleware,
asyncRetryTransactionMiddleware,
authenticate,
videoImportAddValidator,
videoImportDeleteValidator
} from '../../../middlewares'
import { CONFIG, IMAGE_MIMETYPE_EXT, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE } from '../../../initializers'
import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl'
import { createReqFiles } from '../../../helpers/express-utils'
import { logger } from '../../../helpers/logger'
import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
import { VideoModel } from '../../../models/video/video'
import { getVideoActivityPubUrl } from '../../../lib/activitypub'
import { TagModel } from '../../../models/video/tag'
import { VideoImportModel } from '../../../models/video/video-import'
import { JobQueue } from '../../../lib/job-queue/job-queue'
import { processImage } from '../../../helpers/image-utils'
import { join } from 'path'

const auditLogger = auditLoggerFactory('video-imports')
const videoImportsRouter = express.Router()

const reqVideoFileImport = createReqFiles(
[ 'thumbnailfile', 'previewfile' ],
IMAGE_MIMETYPE_EXT,
{
thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
previewfile: CONFIG.STORAGE.PREVIEWS_DIR
}
)

videoImportsRouter.post('/imports',
authenticate,
reqVideoFileImport,
asyncMiddleware(videoImportAddValidator),
asyncRetryTransactionMiddleware(addVideoImport)
)

videoImportsRouter.delete('/imports/:id',
authenticate,
videoImportDeleteValidator,
asyncRetryTransactionMiddleware(deleteVideoImport)
)

// ---------------------------------------------------------------------------

export {
videoImportsRouter
}

// ---------------------------------------------------------------------------

async function addVideoImport (req: express.Request, res: express.Response) {
const body: VideoImportCreate = req.body
const targetUrl = body.targetUrl

let youtubeDLInfo: YoutubeDLInfo
try {
youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
} catch (err) {
logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })

return res.status(400).json({
error: 'Cannot fetch remote information of this URL.'
}).end()
}

// Create video DB object
const videoData = {
name: body.name || youtubeDLInfo.name,
remote: false,
category: body.category || youtubeDLInfo.category,
licence: body.licence || youtubeDLInfo.licence,
language: undefined,
commentsEnabled: body.commentsEnabled || true,
waitTranscoding: body.waitTranscoding || false,
state: VideoState.TO_IMPORT,
nsfw: body.nsfw || youtubeDLInfo.nsfw || false,
description: body.description || youtubeDLInfo.description,
support: body.support || null,
privacy: body.privacy || VideoPrivacy.PRIVATE,
duration: 0, // duration will be set by the import job
channelId: res.locals.videoChannel.id
}
const video = new VideoModel(videoData)
video.url = getVideoActivityPubUrl(video)

// Process thumbnail file?
const thumbnailField = req.files['thumbnailfile']
let downloadThumbnail = true
if (thumbnailField) {
const thumbnailPhysicalFile = thumbnailField[ 0 ]
await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
downloadThumbnail = false
}

// Process preview file?
const previewField = req.files['previewfile']
let downloadPreview = true
if (previewField) {
const previewPhysicalFile = previewField[0]
await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
downloadPreview = false
}

const videoImport: VideoImportModel = await sequelizeTypescript.transaction(async t => {
const sequelizeOptions = { transaction: t }

// Save video object in database
const videoCreated = await video.save(sequelizeOptions)
videoCreated.VideoChannel = res.locals.videoChannel

// Set tags to the video
if (youtubeDLInfo.tags !== undefined) {
const tagInstances = await TagModel.findOrCreateTags(youtubeDLInfo.tags, t)

await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
videoCreated.Tags = tagInstances
}

// Create video import object in database
const videoImport = await VideoImportModel.create({
targetUrl,
state: VideoImportState.PENDING,
videoId: videoCreated.id
}, sequelizeOptions)

videoImport.Video = videoCreated

return videoImport
})

// Create job to import the video
const payload = {
type: 'youtube-dl' as 'youtube-dl',
videoImportId: videoImport.id,
thumbnailUrl: youtubeDLInfo.thumbnailUrl,
downloadThumbnail,
downloadPreview
}
await JobQueue.Instance.createJob({ type: 'video-import', payload })

return res.json(videoImport.toFormattedJSON())
}

async function deleteVideoImport (req: express.Request, res: express.Response) {
// TODO: delete video import
}

+ 2
- 1
server/controllers/api/videos/index.ts View File

@@ -54,6 +54,7 @@ import { VideoFilter } from '../../../../shared/models/videos/video-query.type'
import { createReqFiles, buildNSFWFilter } from '../../../helpers/express-utils'
import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
import { videoCaptionsRouter } from './captions'
import { videoImportsRouter } from './import'

const auditLogger = auditLoggerFactory('videos')
const videosRouter = express.Router()
@@ -81,6 +82,7 @@ videosRouter.use('/', blacklistRouter)
videosRouter.use('/', rateVideoRouter)
videosRouter.use('/', videoCommentRouter)
videosRouter.use('/', videoCaptionsRouter)
videosRouter.use('/', videoImportsRouter)

videosRouter.get('/categories', listVideoCategories)
videosRouter.get('/licences', listVideoLicences)
@@ -160,7 +162,6 @@ async function addVideo (req: express.Request, res: express.Response) {
const videoData = {
name: videoInfo.name,
remote: false,
extname: extname(videoPhysicalFile.filename),
category: videoInfo.category,
licence: videoInfo.licence,
language: videoInfo.language,


+ 1
- 1
server/helpers/custom-validators/activitypub/videos.ts View File

@@ -45,7 +45,7 @@ function isActivityPubVideoDurationValid (value: string) {
}

function sanitizeAndCheckVideoTorrentObject (video: any) {
if (video.type !== 'Video') return false
if (!video || video.type !== 'Video') return false

if (!setValidRemoteTags(video)) return false
if (!setValidRemoteVideoUrls(video)) return false


+ 30
- 0
server/helpers/custom-validators/video-imports.ts View File

@@ -0,0 +1,30 @@
import 'express-validator'
import 'multer'
import * as validator from 'validator'
import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers'
import { exists } from './misc'

function isVideoImportTargetUrlValid (url: string) {
const isURLOptions = {
require_host: true,
require_tld: true,
require_protocol: true,
require_valid_protocol: true,
protocols: [ 'http', 'https' ]
}

return exists(url) &&
validator.isURL('' + url, isURLOptions) &&
validator.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL)
}

function isVideoImportStateValid (value: any) {
return exists(value) && VIDEO_IMPORT_STATES[ value ] !== undefined
}

// ---------------------------------------------------------------------------

export {
isVideoImportStateValid,
isVideoImportTargetUrlValid
}

+ 1
- 1
server/helpers/logger.ts View File

@@ -22,7 +22,7 @@ function loggerReplacer (key: string, value: any) {
}

const consoleLoggerFormat = winston.format.printf(info => {
let additionalInfos = JSON.stringify(info.meta, loggerReplacer, 2)
let additionalInfos = JSON.stringify(info.meta || info.err, loggerReplacer, 2)
if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = ''
else additionalInfos = ' ' + additionalInfos



+ 142
- 0
server/helpers/youtube-dl.ts View File

@@ -0,0 +1,142 @@
import * as youtubeDL from 'youtube-dl'
import { truncate } from 'lodash'
import { CONFIG, CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
import { join } from 'path'
import * as crypto from 'crypto'
import { logger } from './logger'

export type YoutubeDLInfo = {
name: string
description: string
category: number
licence: number
nsfw: boolean
tags: string[]
thumbnailUrl: string
}

function getYoutubeDLInfo (url: string): Promise<YoutubeDLInfo> {
return new Promise<YoutubeDLInfo>((res, rej) => {
const options = [ '-j', '--flat-playlist' ]

youtubeDL.getInfo(url, options, (err, info) => {
if (err) return rej(err)

const obj = normalizeObject(info)

return res(buildVideoInfo(obj))
})
})
}

function downloadYoutubeDLVideo (url: string) {
const hash = crypto.createHash('sha256').update(url).digest('base64')
const path = join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')

logger.info('Importing video %s', url)

const options = [ '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best', '-o', path ]

return new Promise<string>((res, rej) => {
youtubeDL.exec(url, options, async (err, output) => {
if (err) return rej(err)

return res(path)
})
})
}

// ---------------------------------------------------------------------------

export {
downloadYoutubeDLVideo,
getYoutubeDLInfo
}

// ---------------------------------------------------------------------------

function normalizeObject (obj: any) {
const newObj: any = {}

for (const key of Object.keys(obj)) {
// Deprecated key
if (key === 'resolution') continue

const value = obj[key]

if (typeof value === 'string') {
newObj[key] = value.normalize()
} else {
newObj[key] = value
}
}

return newObj
}

function buildVideoInfo (obj: any) {
return {
name: titleTruncation(obj.title),
description: descriptionTruncation(obj.description),
category: getCategory(obj.categories),
licence: getLicence(obj.license),
nsfw: isNSFW(obj),
tags: getTags(obj.tags),
thumbnailUrl: obj.thumbnail || undefined
}
}

function titleTruncation (title: string) {
return truncate(title, {
'length': CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
'separator': /,? +/,
'omission': ' […]'
})
}

function descriptionTruncation (description: string) {
if (!description) return undefined

return truncate(description, {
'length': CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
'separator': /,? +/,
'omission': ' […]'
})
}

function isNSFW (info: any) {
return info.age_limit && info.age_limit >= 16
}

function getTags (tags: any) {
if (Array.isArray(tags) === false) return []

return tags
.filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
.map(t => t.normalize())
.slice(0, 5)
}

function getLicence (licence: string) {
if (!licence) return undefined

if (licence.indexOf('Creative Commons Attribution licence') !== -1) return 1

return undefined
}

function getCategory (categories: string[]) {
if (!categories) return undefined

const categoryString = categories[0]
if (!categoryString || typeof categoryString !== 'string') return undefined

if (categoryString === 'News & Politics') return 11

for (const key of Object.keys(VIDEO_CATEGORIES)) {
const category = VIDEO_CATEGORIES[key]
if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
}

return undefined
}

+ 16
- 2
server/initializers/constants.ts View File

@@ -8,6 +8,7 @@ import { VideoPrivacy } from '../../shared/models/videos'
import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
import { invert } from 'lodash'
import { VideoImportState } from '../../shared/models/videos/video-import-state.enum'

// Use a variable to reload the configuration if we need
let config: IConfig = require('config')
@@ -85,6 +86,7 @@ const JOB_ATTEMPTS: { [ id in JobType ]: number } = {
'activitypub-follow': 5,
'video-file-import': 1,
'video-file': 1,
'video-import': 1,
'email': 5
}
const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
@@ -94,6 +96,7 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
'activitypub-follow': 3,
'video-file-import': 1,
'video-file': 1,
'video-import': 1,
'email': 5
}
const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job
@@ -248,6 +251,9 @@ const CONSTRAINTS_FIELDS = {
}
}
},
VIDEO_IMPORTS: {
URL: { min: 3, max: 2000 } // Length
},
VIDEOS: {
NAME: { min: 3, max: 120 }, // Length
LANGUAGE: { min: 1, max: 10 }, // Length
@@ -262,7 +268,7 @@ const CONSTRAINTS_FIELDS = {
},
EXTNAME: [ '.mp4', '.ogv', '.webm' ],
INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2
DURATION: { min: 1 }, // Number
DURATION: { min: 0 }, // Number
TAGS: { min: 0, max: 5 }, // Number of total tags
TAG: { min: 2, max: 30 }, // Length
THUMBNAIL: { min: 2, max: 30 },
@@ -363,7 +369,14 @@ const VIDEO_PRIVACIES = {

const VIDEO_STATES = {
[VideoState.PUBLISHED]: 'Published',
[VideoState.TO_TRANSCODE]: 'To transcode'
[VideoState.TO_TRANSCODE]: 'To transcode',
[VideoState.TO_IMPORT]: 'To import'
}

const VIDEO_IMPORT_STATES = {
[VideoImportState.FAILED]: 'Failed',
[VideoImportState.PENDING]: 'Pending',
[VideoImportState.SUCCESS]: 'Success'
}

const VIDEO_MIMETYPE_EXT = {
@@ -585,6 +598,7 @@ export {
RATES_LIMIT,
VIDEO_EXT_MIMETYPE,
JOB_COMPLETED_LIFETIME,
VIDEO_IMPORT_STATES,
VIDEO_VIEW_LIFETIME,
buildLanguages
}


+ 3
- 1
server/initializers/database.ts View File

@@ -24,6 +24,7 @@ import { VideoTagModel } from '../models/video/video-tag'
import { CONFIG } from './constants'
import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update'
import { VideoCaptionModel } from '../models/video/video-caption'
import { VideoImportModel } from '../models/video/video-import'

require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string

@@ -81,7 +82,8 @@ async function initDatabaseModels (silent: boolean) {
VideoTagModel,
VideoModel,
VideoCommentModel,
ScheduleVideoUpdateModel
ScheduleVideoUpdateModel,
VideoImportModel
])

// Check extensions exist in the database


+ 129
- 0
server/lib/job-queue/handlers/video-import.ts View File

@@ -0,0 +1,129 @@
import * as Bull from 'bull'
import { logger } from '../../../helpers/logger'
import { downloadYoutubeDLVideo } from '../../../helpers/youtube-dl'
import { VideoImportModel } from '../../../models/video/video-import'
import { VideoImportState } from '../../../../shared/models/videos'
import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffmpeg-utils'
import { extname, join } from 'path'
import { VideoFileModel } from '../../../models/video/video-file'
import { renamePromise, statPromise, unlinkPromise } from '../../../helpers/core-utils'
import { CONFIG, sequelizeTypescript } from '../../../initializers'
import { doRequestAndSaveToFile } from '../../../helpers/requests'
import { VideoState } from '../../../../shared'
import { JobQueue } from '../index'
import { federateVideoIfNeeded } from '../../activitypub'

export type VideoImportPayload = {
type: 'youtube-dl'
videoImportId: number
thumbnailUrl: string
downloadThumbnail: boolean
downloadPreview: boolean
}

async function processVideoImport (job: Bull.Job) {
const payload = job.data as VideoImportPayload
logger.info('Processing video import in job %d.', job.id)

const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId)
if (!videoImport) throw new Error('Cannot import video %s: the video import entry does not exist anymore.')

let tempVideoPath: string
try {
// Download video from youtubeDL
tempVideoPath = await downloadYoutubeDLVideo(videoImport.targetUrl)

// Get information about this video
const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
const fps = await getVideoFileFPS(tempVideoPath)
const stats = await statPromise(tempVideoPath)
const duration = await getDurationFromVideoFile(tempVideoPath)

// Create video file object in database
const videoFileData = {
extname: extname(tempVideoPath),
resolution: videoFileResolution,
size: stats.size,
fps,
videoId: videoImport.videoId
}
const videoFile = new VideoFileModel(videoFileData)

// Move file
const destination = join(CONFIG.STORAGE.VIDEOS_DIR, videoImport.Video.getVideoFilename(videoFile))
await renamePromise(tempVideoPath, destination)

// Process thumbnail
if (payload.downloadThumbnail) {
if (payload.thumbnailUrl) {
const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destThumbnailPath)
} else {
await videoImport.Video.createThumbnail(videoFile)
}
}

// Process preview
if (payload.downloadPreview) {
if (payload.thumbnailUrl) {
const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
await doRequestAndSaveToFile({ method: 'GET', uri: payload.thumbnailUrl }, destPreviewPath)
} else {
await videoImport.Video.createPreview(videoFile)
}
}

// Create torrent
await videoImport.Video.createTorrentAndSetInfoHash(videoFile)

const videoImportUpdated: VideoImportModel = await sequelizeTypescript.transaction(async t => {
await videoFile.save({ transaction: t })

// Update video DB object
videoImport.Video.duration = duration
videoImport.Video.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
const videoUpdated = await videoImport.Video.save({ transaction: t })

// Now we can federate the video
await federateVideoIfNeeded(videoImport.Video, true, t)

// Update video import object
videoImport.state = VideoImportState.SUCCESS
const videoImportUpdated = await videoImport.save({ transaction: t })

logger.info('Video %s imported.', videoImport.targetUrl)

videoImportUpdated.Video = videoUpdated
return videoImportUpdated
})

// Create transcoding jobs?
if (videoImportUpdated.Video.state === VideoState.TO_TRANSCODE) {
// Put uuid because we don't have id auto incremented for now
const dataInput = {
videoUUID: videoImportUpdated.Video.uuid,
isNewVideo: true
}

await JobQueue.Instance.createJob({ type: 'video-file', payload: dataInput })
}

} catch (err) {
try {
if (tempVideoPath) await unlinkPromise(tempVideoPath)
} catch (errUnlink) {
logger.error('Cannot cleanup files after a video import error.', { err: errUnlink })
}

videoImport.state = VideoImportState.FAILED
await videoImport.save()

throw err
}
}

// ---------------------------------------------------------------------------

export {
processVideoImport
}

+ 7
- 3
server/lib/job-queue/job-queue.ts View File

@@ -9,6 +9,7 @@ import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './
import { EmailPayload, processEmail } from './handlers/email'
import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file'
import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
import { processVideoImport, VideoImportPayload } from './handlers/video-import'

type CreateJobArgument =
{ type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -17,7 +18,8 @@ type CreateJobArgument =
{ type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
{ type: 'video-file-import', payload: VideoFileImportPayload } |
{ type: 'video-file', payload: VideoFilePayload } |
{ type: 'email', payload: EmailPayload }
{ type: 'email', payload: EmailPayload } |
{ type: 'video-import', payload: VideoImportPayload }

const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
'activitypub-http-broadcast': processActivityPubHttpBroadcast,
@@ -26,7 +28,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
'activitypub-follow': processActivityPubFollow,
'video-file-import': processVideoFileImport,
'video-file': processVideoFile,
'email': processEmail
'email': processEmail,
'video-import': processVideoImport
}

const jobsWithRequestTimeout: { [ id in JobType ]?: boolean } = {
@@ -43,7 +46,8 @@ const jobTypes: JobType[] = [
'activitypub-http-unicast',
'email',
'video-file',
'video-file-import'
'video-file-import',
'video-import'
]

class JobQueue {


+ 1
- 0
server/middlewares/validators/index.ts View File

@@ -11,3 +11,4 @@ export * from './video-blacklist'
export * from './video-channels'
export * from './webfinger'
export * from './search'
export * from './video-imports'

+ 51
- 0
server/middlewares/validators/video-imports.ts View File

@@ -0,0 +1,51 @@
import * as express from 'express'
import { body, param } from 'express-validator/check'
import { isIdValid } from '../../helpers/custom-validators/misc'
import { logger } from '../../helpers/logger'
import { areValidationErrors } from './utils'
import { getCommonVideoAttributes } from './videos'
import { isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
import { cleanUpReqFiles } from '../../helpers/utils'
import { isVideoChannelOfAccountExist, isVideoNameValid } from '../../helpers/custom-validators/videos'

const videoImportAddValidator = getCommonVideoAttributes().concat([
body('targetUrl').custom(isVideoImportTargetUrlValid).withMessage('Should have a valid video import target URL'),
body('channelId')
.toInt()
.custom(isIdValid).withMessage('Should have correct video channel id'),
body('name')
.optional()
.custom(isVideoNameValid).withMessage('Should have a valid name'),

async (req: express.Request, res: express.Response, next: express.NextFunction) => {
logger.debug('Checking videoImportAddValidator parameters', { parameters: req.body })

const user = res.locals.oauth.token.User

if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)