使用Deoldify模型来对视频影像复原（对运行时遇到的问题的做出的解决方案）

1.模型权重

第一次做Deoldify模型的复原，由于对该模型的使用不太清晰，搜索了一篇文章来查看，文章如下：https://blog.csdn.net/weixin_42512684/article/details/117376885  

文章发布于21年，但是其作者提供的代码也是帮助解决了很多问题，但是由于作者所提供的模型如stable类型的模型已经无法下载了，这里贴出我收集的三类模型。

处理视频文件的话，使用Video权重就行。

模型权重（百度网盘）：

https://pan.baidu.com/s/1Oadr1qk6vWQpokFwsprH8Q?pwd=iybk

2.代码问题：

先去这个网址下获取项目的压缩包：https://github.com/jantic/DeOldify

解压压缩包到对应项目位置。

pip install -r requirements.txt

首先，先下载所需要的各种包，当然这个包不齐全会缺少好几个包，需要后续根据缺少的名字来pip install 一下，不知道包的pip指令可以在网上查询。

直接使用该作者之前提供的运行的代码遇到了这个问题：

修改运行的代码如下即可解决问题：

from deoldify import device
from deoldify.device_id import DeviceId

#choices:  CPU, GPU0...GPU7
device.set(device=DeviceId.GPU0)

from deoldify.visualize import *
plt.style.use('dark_background')
import warnings
warnings.filterwarnings("ignore", category=UserWarning, message=".*?Your .*? set is empty.*?")

colorizer = get_video_colorizer(workfolder='.')


#NOTE:  Max is 44 with 11GB video cards.  21 is a good default
render_factor= 21

file_name = 'video'

file_name_ext = file_name + '.mp4'
# result_path = Path('E:./video_result.mp4')

def progress_bar(progress, message):
    print(f"{message}: {progress}%")
colorizer.colorize_from_file_name(file_name_ext, render_factor=render_factor, g_process_bar=progress_bar)
# colorizer.save_colorized_video(Path(file_name_ext), result_path)

再往下运行就会遇到一种同一类但需要修改的地方十分多的问题，如下所示，还有其它bug，但是没必要一一列出：

 File "E:\***\******\sd-webui-deoldify-main\deoldify\visualize.py", line 237, in _extract_raw_frames
    bwframes_folder = self.bwframes_root / (source_path.stem)
                      ~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~
TypeError: unsupported operand type(s) for /: 'str' and 'str'

可以发现visualize.py存在诸多需要修改的地方，我将修改好的visualize.py贴在下方，这是我个人的一个修改方式：

from fastai.core import *
from fastai.vision import *
from matplotlib.axes import Axes
from .filters import IFilter, MasterFilter, ColorizerFilter
from .generators import gen_inference_deep, gen_inference_wide
from PIL import Image
import ffmpeg
import yt_dlp as youtube_dl
import gc
import requests
from io import BytesIO
import base64
import cv2
import logging
import gradio as gr
from pathlib import Path

class ModelImageVisualizer:
    def __init__(self, filter: IFilter, results_dir: str = None):
        self.filter = filter
        self.results_dir = None if results_dir is None else Path(results_dir)
        self.results_dir.mkdir(parents=True, exist_ok=True)

    def _clean_mem(self):
        torch.cuda.empty_cache()
        # gc.collect()

    def _open_pil_image(self, path: Path) -> Image:
        return PIL.Image.open(path).convert('RGB')

    def _get_image_from_url(self, url: str) -> Image:
        response = requests.get(url, timeout=30, headers={'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'})
        img = PIL.Image.open(BytesIO(response.content)).convert('RGB')
        return img

    def plot_transformed_image_from_url(
        self,
        url: str,
        path: str = 'test_images/image.png',
        results_dir:Path = None,
        figsize: Tuple[int, int] = (20, 20),
        render_factor: int = None,
        
        display_render_factor: bool = False,
        compare: bool = False,
        post_process: bool = True,
    ) -> Path:
        img = self._get_image_from_url(url)
        img.save(path)
        return self.plot_transformed_image(
            path=path,
            results_dir=results_dir,
            figsize=figsize,
            render_factor=render_factor,
            display_render_factor=display_render_factor,
            compare=compare,
            post_process = post_process,
        )

    def plot_transformed_image(
        self,
        path: str,
        results_dir:Path = None,
        figsize: Tuple[int, int] = (20, 20),
        render_factor: int = None,
        display_render_factor: bool = False,
        compare: bool = False,
        post_process: bool = True,
    ) -> Path:
        path = Path(path)
        if results_dir is None:
            results_dir = Path(self.results_dir)
        result = self.get_transformed_image(
            path, render_factor, post_process=post_process
        )
        orig = self._open_pil_image(path)
        if compare:
            self._plot_comparison(
                figsize, render_factor, display_render_factor, orig, result
            )
        else:
            self._plot_solo(figsize, render_factor, display_render_factor, result)

        orig.close()
        result_path = self._save_result_image(path, result, results_dir=results_dir)
        result.close()
        return result_path

    def _plot_comparison(
        self,
        figsize: Tuple[int, int],
        render_factor: int,
        display_render_factor: bool,
        orig: Image,
        result: Image,
    ):
        fig, axes = plt.subplots(1, 2, figsize=figsize)
        self._plot_image(
            orig,
            axes=axes[0],
            figsize=figsize,
            render_factor=render_factor,
            display_render_factor=False,
        )
        self._plot_image(
            result,
            axes=axes[1],
            figsize=figsize,
            render_factor=render_factor,
            display_render_factor=display_render_factor,
        )

    def _plot_solo(
        self,
        figsize: Tuple[int, int],
        render_factor: int,
        display_render_factor: bool,
        result: Image,
    ):
        fig, axes = plt.subplots(1, 1, figsize=figsize)
        self._plot_image(
            result,
            axes=axes,
            figsize=figsize,
            render_factor=render_factor,
            display_render_factor=display_render_factor,
        )

    def _save_result_image(self, source_path: Path, image: Image, results_dir = None) -> Path:
        if results_dir is None:
            results_dir = Path(self.results_dir)
        result_path = results_dir / source_path.name
        image.save(result_path)
        return result_path

    def get_transformed_image(
        self, path: Path, render_factor: int = None, post_process: bool = True,
    ) -> Image:
        self._clean_mem()
        orig_image = self._open_pil_image(path)
        filtered_image = self.filter.filter(
            orig_image, orig_image, render_factor=render_factor,post_process=post_process
        )

        return filtered_image
    
    # 直接从图片转换
    def get_transformed_image_from_image(
        self, image: Image, render_factor: int = None, post_process: bool = True,
    ) -> Image:
        self._clean_mem()
        orig_image = image
        filtered_image = self.filter.filter(
            orig_image, orig_image, render_factor=render_factor,post_process=post_process
        )

        return filtered_image

    def _plot_image(
        self,
        image: Image,
        render_factor: int,
        axes: Axes = None,
        figsize=(20, 20),
        display_render_factor = False,
    ):
        if axes is None:
            _, axes = plt.subplots(figsize=figsize)
        axes.imshow(np.asarray(image) / 255)
        axes.axis('off')
        if render_factor is not None and display_render_factor:
            plt.text(
                10,
                10,
                'render_factor: ' + str(render_factor),
                color='white',
                backgroundcolor='black',
            )

    def _get_num_rows_columns(self, num_images: int, max_columns: int) -> Tuple[int, int]:
        columns = min(num_images, max_columns)
        rows = num_images // columns
        rows = rows if rows * columns == num_images else rows + 1
        return rows, columns

import os
class VideoColorizer:
    def __init__(self, vis: ModelImageVisualizer,workfolder: Path = None):
        self.vis = vis
        self.workfolder = workfolder
        self.source_folder = os.path.join(self.workfolder, "source")
        self.bwframes_root = os.path.join(self.workfolder, "bwframes")
        self.audio_root = os.path.join(self.workfolder, "audio")
        self.colorframes_root = os.path.join(self.workfolder,  "colorframes")
        self.result_folder = os.path.join(self.workfolder,  "result")

    def _purge_images(self, dir):
        for f in os.listdir(dir):
            if re.search('.*?\.jpg', f):
                os.remove(os.path.join(dir, f))

    def _get_ffmpeg_probe(self, path:Path):
        try:
            probe = ffmpeg.probe(str(path))
            return probe
        except ffmpeg.Error as e:
            logging.error("ffmpeg error: {0}".format(e), exc_info=True)
            logging.error('stdout:' + e.stdout.decode('UTF-8'))
            logging.error('stderr:' + e.stderr.decode('UTF-8'))
            raise e
        except Exception as e:
            logging.error('Failed to instantiate ffmpeg.probe.  Details: {0}'.format(e), exc_info=True)   
            raise e

    def _get_fps(self, source_path: Path) -> str:
        probe = self._get_ffmpeg_probe(source_path)
        stream_data = next(
            (stream for stream in probe['streams'] if stream['codec_type'] == 'video'),
            None,
        )
        return stream_data['avg_frame_rate']

    def _download_video_from_url(self, source_url, source_path: Path):
        if source_path.exists():
            source_path.unlink()

        ydl_opts = {
            'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4',
            'outtmpl': str(source_path),
            'retries': 30,
            'fragment-retries': 30
        }
        with youtube_dl.YoutubeDL(ydl_opts) as ydl:
            ydl.download([source_url])

    def _extract_raw_frames(self, source_path: Path):
        bwframes_folder = os.path.join(self.bwframes_root, source_path.stem)
        bwframes_folder = Path(bwframes_folder)
        bwframe_path_template = os.path.join(str(bwframes_folder), '%5d.jpg')
        bwframes_folder.mkdir(parents=True, exist_ok=True)
        self._purge_images(bwframes_folder)

        process = (
            ffmpeg
                .input(str(source_path))
                .output(str(bwframe_path_template), format='image2', vcodec='mjpeg', **{'q:v':'0'})
                .global_args('-hide_banner')
                .global_args('-nostats')
                .global_args('-loglevel', 'error')
        )

        try:
            process.run()
        except ffmpeg.Error as e:
            logging.error("ffmpeg error: {0}".format(e), exc_info=True)
            logging.error('stdout:' + e.stdout.decode('UTF-8'))
            logging.error('stderr:' + e.stderr.decode('UTF-8'))
            raise e
        except Exception as e:
            logging.error('Errror while extracting raw frames from source video.  Details: {0}'.format(e), exc_info=True)   
            raise e

    def _colorize_raw_frames(
        self, source_path: Path, render_factor: int = None, post_process: bool = True,g_process_bar: gr.Progress = None
    ):
        colorframes_folder = Path(self.colorframes_root) / source_path.stem
        colorframes_folder.mkdir(parents=True, exist_ok=True)
        self._purge_images(colorframes_folder)
        bwframes_folder = Path(self.bwframes_root) / source_path.stem
        p_status = 0
        image_index = 0
        total_images = len(os.listdir(str(bwframes_folder)))
        for img in progress_bar(os.listdir(str(bwframes_folder))):
            img_path = bwframes_folder / img

            image_index += 1
            if g_process_bar is not None:
                p_status = image_index / total_images
                g_process_bar(p_status,"Colorizing...")

            if os.path.isfile(str(img_path)):
                color_image = self.vis.get_transformed_image(
                    str(img_path), render_factor=render_factor, post_process=post_process
                )
                color_image.save(str(colorframes_folder / img))

    def _build_video(self, source_path: Path) -> Path:
        colorized_path = Path(self.result_folder) / source_path.name
        colorframes_folder = Path(self.colorframes_root) / source_path.stem

        colorframes_path_template = str(colorframes_folder / '%5d.jpg')
        colorized_path.parent.mkdir(parents=True, exist_ok=True)
        if colorized_path.exists():
            colorized_path.unlink()
        fps = self._get_fps(source_path)

        process = (
            ffmpeg 
                .input(str(colorframes_path_template), format='image2', vcodec='mjpeg', framerate=fps) 
                .output(str(colorized_path), crf=17, vcodec='libx264')
                .global_args('-hide_banner')
                .global_args('-nostats')
                .global_args('-loglevel', 'error')
        )

        try:
            process.run()
        except ffmpeg.Error as e:
            logging.error("ffmpeg error: {0}".format(e), exc_info=True)
            logging.error('stdout:' + e.stdout.decode('UTF-8'))
            logging.error('stderr:' + e.stderr.decode('UTF-8'))
            raise e
        except Exception as e:
            logging.error('Errror while building output video.  Details: {0}'.format(e), exc_info=True)   
            raise e

        result_path = Path(self.result_folder) / source_path.name
        if result_path.exists():
            result_path.unlink()
        # making copy of non-audio version in case adding back audio doesn't apply or fails.
        shutil.copyfile(str(colorized_path), str(result_path))

        # adding back sound here
        audio_file = Path(str(source_path).replace('.mp4', '.aac'))
        if audio_file.exists():
            audio_file.unlink()

        os.system(
            'ffmpeg -y -i "'
            + str(source_path)
            + '" -vn -acodec copy "'
            + str(audio_file)
            + '"'
            + ' -hide_banner'
            + ' -nostats'
            + ' -loglevel error'
        )

        if audio_file.exists():
            os.system(
                'ffmpeg -y -i "'
                + str(colorized_path)
                + '" -i "'
                + str(audio_file)
                + '" -shortest -c:v copy -c:a aac -b:a 256k "'
                + str(result_path)
                + '"'
                + ' -hide_banner'
                + ' -nostats'
                + ' -loglevel error'
            )
        logging.info('Video created here: ' + str(result_path))
        return result_path

    def colorize_from_url(
        self,
        source_url,
        file_name: str,
        render_factor: int = None,
        post_process: bool = True,

    ) -> Path:
        source_path = Path(self.source_folder) / file_name

        self._download_video_from_url(source_url, source_path)
        return self._colorize_from_path(
            source_path, render_factor=render_factor, post_process=post_process
        )

    def colorize_from_file_name(
        self, file_name: str, render_factor: int = None, post_process: bool = True,g_process_bar: gr.Progress = None
    ) -> Path:
        source_path = Path(self.source_folder) / file_name
        return self._colorize_from_path(
            source_path, render_factor=render_factor,  post_process=post_process,g_process_bar=g_process_bar
        )

    def _colorize_from_path(
        self, source_path: Path, render_factor: int = None, post_process: bool = True,g_process_bar: gr.Progress = None
    ) -> Path:
        if not source_path.exists():
            raise Exception(
                'Video at path specfied, ' + str(source_path) + ' could not be found.'
            )
        g_process_bar(0,"Extracting frames...")
        self._extract_raw_frames(source_path)
        self._colorize_raw_frames(
            source_path, render_factor=render_factor,post_process=post_process,g_process_bar=g_process_bar
        )
        return self._build_video(source_path)


def get_video_colorizer(render_factor: int = 21,workfolder:str = "./video") -> VideoColorizer:
    return get_stable_video_colorizer(render_factor=render_factor,workfolder=workfolder)


def get_artistic_video_colorizer(
    root_folder: Path = Path('./'),
    weights_name: str = 'ColorizeArtistic_gen',
    results_dir='result_images',
    render_factor: int = 35
) -> VideoColorizer:
    learn = gen_inference_deep(root_folder=root_folder, weights_name=weights_name)
    filtr = MasterFilter([ColorizerFilter(learn=learn)], render_factor=render_factor)
    vis = ModelImageVisualizer(filtr, results_dir=results_dir)
    return VideoColorizer(vis)


def get_stable_video_colorizer(
    root_folder: Path = Path('./'),
    weights_name: str = 'ColorizeVideo_gen',
    results_dir='result_images',
    render_factor: int = 21,
    workfolder:str = "./video"
) -> VideoColorizer:
    learn = gen_inference_wide(root_folder=root_folder, weights_name=weights_name)
    filtr = MasterFilter([ColorizerFilter(learn=learn)], render_factor=render_factor)
    vis = ModelImageVisualizer(filtr, results_dir=results_dir)
    return VideoColorizer(vis,workfolder=workfolder)


def get_image_colorizer(
    root_folder: Path = Path('./'), render_factor: int = 35, artistic: bool = True
) -> ModelImageVisualizer:
    if artistic:
        return get_artistic_image_colorizer(root_folder=root_folder, render_factor=render_factor)
    else:
        return get_stable_image_colorizer(root_folder=root_folder, render_factor=render_factor)


def get_stable_image_colorizer(
    root_folder: Path = Path('./'),
    weights_name: str = 'ColorizeStable_gen',
    results_dir='result_images',
    render_factor: int = 35
) -> ModelImageVisualizer:
    learn = gen_inference_wide(root_folder=root_folder, weights_name=weights_name)
    filtr = MasterFilter([ColorizerFilter(learn=learn)], render_factor=render_factor)
    vis = ModelImageVisualizer(filtr, results_dir=results_dir)
    return vis


def get_artistic_image_colorizer(
    root_folder: Path = Path('./'),
    weights_name: str = 'ColorizeArtistic_gen',
    results_dir='result_images',
    render_factor: int = 35
) -> ModelImageVisualizer:
    learn = gen_inference_deep(root_folder=root_folder, weights_name=weights_name)
    filtr = MasterFilter([ColorizerFilter(learn=learn)], render_factor=render_factor)
    vis = ModelImageVisualizer(filtr, results_dir=results_dir)
    return vis

修改好后，代码便可顺利运行。

3.运行和后续视频处理

我们可以发现Deolidfy是将视频处理为一帧一帧的jpg文件再进行颜色恢复，然而呢，这个代码，貌似并不能自动的将图片合成为视频，在运行完成后会出现如下的问题：

虽然坏消息是不能直接一套代码合成为视频，但是好消息是其处理完成后的图片会自动生成文件夹报存下来，保存的文件夹为"E:\***\***\colorframes\video"

而未被处理的图像帧会报存在"E:\***\***\bwframes\video"

我们可以通过一个代码将处理出来的图片们融合为视频然后截出原来视频的音轨将其融合为一个视频就可以了。实现这个融合功能的代码如下：

from moviepy.editor import VideoFileClip, ImageSequenceClip
import os
# 设置文件夹路径和视频源路径
folder_path = 'E:/yolo/pythonProject/colorframes/video'
video_source_path = 'E:/yolo/pythonProject/source/video.mp4'

# 从文件夹中的图像序列创建视频剪辑
image_files = [f for f in os.listdir(folder_path) if f.endswith('.jpg')]
image_files.sort()
clip = ImageSequenceClip([os.path.join(folder_path, img) for img in image_files], fps=30)

# 从视频源文件中提取音频剪辑
video_clip = VideoFileClip(video_source_path)
audio_clip = video_clip.audio

# 将图像视频剪辑与音频剪辑合成为最终视频剪辑
final_clip = clip.set_audio(audio_clip)

# 保存最终视频剪辑
final_clip.write_videofile('output_with_audio.mp4', codec='libx264', audio_codec='aac')

总结：

将处理好的视频进行对比，不难发现deoldify模型依旧具有很强的复原能力，但是仍然会有一些地方出现原片中黄色的斑驳，而且主要出现于人类的脸部和视频中环境较为复杂的地方，如果在训练的时候多加入这类的图片，也许模型可能还能再有提升的空间。

效果对比：文章来源地址https://www.toymoban.com/news/detail-861778.html

到了这里，关于使用Deoldify模型来对视频影像复原（对运行时遇到的问题的做出的解决方案）的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！