MINIO服务器基于AWS S3 SDK 文件分片上传及下载（C++实现）

这篇具有很好参考价值的文章主要介绍了MINIO服务器基于AWS S3 SDK 文件分片上传及下载（C++实现）。希望对大家有所帮助。如果存在错误或未考虑完全的地方，请大家不吝赐教，您也可以点击"举报违法"按钮提交疑问。

基于aws-s3安装

安装

安装环境依赖：

Debian/Ubuntu-based systems : sudo apt-get install libcurl4-openssl-dev
libssl-dev uuid-dev zlib1g-dev libpulse-dev cmake
Redhat/Fedora-based systems : sudo dnf install libcurl-devel openssl-devel
libuuid-devel pulseaudio-devel cmake

获取SDK源码并安装：

git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git
cd aws-sdk-cpp
mkdir build
cd build
cmake ..
make -j8
sudo make install

项目中CMakeLists.txt配置：

cmake_minimum_required(VERSION 3.19)
project(project_name)
set(CMAKE_CXX_STANDARD 11)
 
# Find the AWS SDK for C++ package.
find_package(AWSSDK CONFIG COMPONENTS core s3 REQUIRED)
 
include_directories(${AWSSDK_INCLUDE_DIRS})
add_executable(project_name mian.cpp)
target_link_libraries(project_name PRIVATE  aws-cpp-sdk-s3)

初始化S3连接

S3Client InitS3Client(const Aws::String IPPort, const Aws::String& accessKeyId, const Aws::String& secretKey, std::string securityToken){
    // 初始化 S3 Client
    Aws::Client::ClientConfiguration cfg;
    cfg.endpointOverride = IPPort;
    cfg.scheme = Aws::Http::Scheme::HTTP;
    cfg.verifySSL = false;

    Aws::Auth::AWSCredentials cred(accessKeyId, secretKey, securityToken);
    S3Client client(cred, cfg,
                    Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false);
    return client;
}

上传文件

一般文件上传

/// 判断待上传的文件是否存在
inline bool file_exists(const string& name)
{
    struct stat buffer;
    return (stat(name.c_str(), &buffer) == 0);
}

/// 上传文件
bool doPutObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const string& file_name, const Aws::String &region)
{
    // 判断文件是否存在
    if (!file_exists(file_name)) {
        cout << "ERROR: 找不到这个文件，这个文件不存在"
             << endl;
        return false;
    }

    /// 初始化上传请求
    Aws::S3::Model::PutObjectRequest object_request;
    object_request.SetBucket(fromBucket);
    object_request.SetKey(objectKey);

    /// 建立文件输入串流，可为任何串流
    const shared_ptr<Aws::IOStream> input_data =
            Aws::MakeShared<Aws::FStream>("SampleAllocationTag",
                                          file_name.c_str(),
                                          ios_base::in | ios_base::binary);
    object_request.SetBody(input_data);

    /// 开始上传至S3
    Aws::S3::Model::PutObjectOutcome put_object_outcome = client.PutObject(object_request);

    if (!put_object_outcome.IsSuccess()) {
        auto error = put_object_outcome.GetError();
        cout << "ERROR: " << error.GetExceptionName() << ": "
             << error.GetMessage() << endl;
        return false;
    }else {
        cout << "success" << endl;
        return true;
    }
}

/// 启动上传程序
bool StartUpLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){
    // 初始化
    Aws::SDKOptions options;
    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
    Aws::InitAPI(options);

    /// 设置ip、acccessKey、secretKey
    Aws::String IP = Ips;
    Aws::String accessKeyId = acccessKey;
    Aws::String secretKey = secretKeyS;

    S3Client client;
    client = InitS3Client(IP, accessKeyId, secretKey, securityToken);

    Aws::InitAPI(options);
    {
        /// 设置上传的目标桶和文件路径
        const Aws::String bucket_name = bucket_names;
        const Aws::String object_name = object_names;
        if (!doPutObject(object_name, bucket_name, client, imagename, " ")) {
            std::string errorInfo =  object_name + " 上传失败";
            writeLog(L_ERROR, errorInfo.c_str());
            return false;
        }
    }
    writeLog(L_INFO, "====> 文件上传成功");
    return true;
}

分块上传

/// MINIO 切片上传
bool mainCutUpload(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){


    Aws::SDKOptions options;
    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
    Aws::InitAPI(options);

    Aws::String IP = Ips;
    Aws::String accessKeyId = acccessKey;
    Aws::String secretKey = secretKeyS;

    /// 初始化S3服务
    S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken);

    Aws::InitAPI(options);
    {
        /// 设置上传的目标桶和文件路径
        const Aws::String bucket_name = bucket_names;
        const Aws::String object_name = object_names;
        std::string bucket = bucket_name;
        std::string key = object_name;

        /// 初始化分块上传任务
        Aws::S3::Model::CreateMultipartUploadRequest create_request;
        create_request.SetBucket(bucket.c_str());
        create_request.SetKey(key.c_str());
        Aws::S3::Model::CreateMultipartUploadOutcome outcome = client.CreateMultipartUpload(create_request);

        /// 获取上传ID
        std::string upload_id = outcome.GetResult().GetUploadId();
        std::cout << "upload id is:" << upload_id << std::endl;

        /// 创建分段上传输出
        if (!outcome.IsSuccess()) {
            auto err = outcome.GetError();
            std::cout << "Error: CreateMultipartUpload: " <<
                      err.GetExceptionName() << ": " << err.GetMessage() << std::endl;
            return false;
        }
        else
            std::cout << "Success: CreateMultipartUpload: " << bucket << std::endl;

        std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetBucket() << std::endl;
        std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetKey() << std::endl;
        std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetUploadId() << std::endl;

        Aws::String UploadId = upload_id;

        /// 启动分块上传
        Aws::S3::Model::UploadPartRequest request1;
        request1.SetBucket(bucket);
        request1.SetKey(key);

        /// 每个分片大小
        long partSize = 5 * 1024 * 1024;

        /// 读取文件
        std::fstream file(imagename.c_str(),std::ios::in | std::ios::binary);
        file.seekg(0,std::ios::end);
        long fileSize = file.tellg();   /// 获取文件大小
        file.seekg(0, std::ios::beg);
        char* buffer = new char[partSize];  // 定义缓存

        /// 初始化上传变量
        long filePosition = 0;
        Aws::Vector<Aws::S3::Model::CompletedPart> completeParts;
        int partNumber = 1;

        /// 开始上传
        while(filePosition < fileSize){
            partSize = std::min(partSize,(fileSize - filePosition));
            file.read(buffer,partSize);
            Aws::S3::Model::UploadPartRequest uploadPartRequest;
            uploadPartRequest.WithBucket(bucket).WithKey(key).WithUploadId(UploadId).WithPartNumber(partNumber).WithContentLength(partSize);
            Aws::String str(buffer,partSize);
            auto input_data = Aws::MakeShared<Aws::StringStream>("UploadPartStream",str);
            uploadPartRequest.SetBody(input_data);
            filePosition += partSize;
            auto uploadPartResult = client.UploadPart(uploadPartRequest);
            completeParts.push_back(Aws::S3::Model::CompletedPart().WithETag(uploadPartResult.GetResult().GetETag()).WithPartNumber(partNumber));
            memset(buffer, 0, partSize);
            ++partNumber;
        }

        // 完成分块上传
        Aws::S3::Model::CompleteMultipartUploadRequest request2;
        request2.SetBucket(bucket);
        request2.SetKey(key);
        request2.SetUploadId(UploadId);
        Aws::S3::Model::CompletedMultipartUpload completed_multipart_upload;

        completed_multipart_upload.SetParts(completeParts);
        request2.SetMultipartUpload(completed_multipart_upload);
        Aws::S3::Model::CompleteMultipartUploadOutcome outcome3 = client.CompleteMultipartUpload(request2);

        if (outcome3.IsSuccess()) {
            std::cout << "Success: CompleteMultipartUpload " << std::endl;
            return true;
        }
        else
        {
            std::cout << "Error: CompleteMultipartUpload: " <<
                      outcome.GetError().GetMessage() << std::endl;
            return false;
        }

    }
}

下载文件

/// 从MINIO中下载文件或数据
bool doGetObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const Aws::String savePath,const Aws::String &region)
{
    /// 初始化下载请求
    Aws::S3::Model::GetObjectRequest object_request;
    object_request.SetBucket(fromBucket);
    object_request.SetKey(objectKey);

    /// 从S3服务器中下载数据
    Aws::S3::Model::GetObjectOutcome get_object_outcome = client.GetObject(object_request);

    if (get_object_outcome.IsSuccess()) {
        /// 保存文件
        Aws::OFStream local_file;
        local_file.open(savePath, std::ios::out | std::ios::binary);
        local_file << get_object_outcome.GetResultWithOwnership().GetBody().rdbuf();
        std::cout << "Done!" << std::endl;
        return true;
    } else {
        auto err = get_object_outcome.GetError();
        std::cout << "Error: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl;
        return false;
    }
}


/// 启动下载程序
bool StartDownLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_name, std::string downPath, std::string imagePath, std::string securityToken,
    bool &flags){
    // 初始化Aws API
    Aws::SDKOptions options;
    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
    Aws::InitAPI(options);

    /// 设置ip、acccessKey、secretKey
    Aws::String IP = Ips;
    Aws::String accessKeyId = acccessKey;
    Aws::String secretKey = secretKeyS;

    /// 初始化S3服务
    S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken);

    /// 根据get接口的桶名称设置文件所在的桶
    Aws::S3::Model::Bucket temps{};
    std::string stringK = bucket_name;
    temps.SetName(stringK);

    /// 查找MINIO相关数据
    Aws::S3::Model::ListObjectsRequest objects_request;
    objects_request.WithBucket(temps.GetName());   /// 设置桶路径
    objects_request.SetPrefix(downPath);   /// 设置文件夹路径

    auto list_objects_outcome = client.ListObjects(objects_request);  /// 判断存储桶及相关连接是否有效

    if (flags){
        writeLog(L_INFO, "====> 文件下载终止");
        return false;
    }
    if (list_objects_outcome.IsSuccess())
    {
        /// 获取该目录下的前一千个文件
        Aws::Vector<Aws::S3::Model::Object> object_list =
                list_objects_outcome.GetResult().GetContents();

        if (object_list.size() == 1000){

            /// 当下载数量大于一千时 循环下载文件
            while(object_list.size() == 1000){
                auto finally_object = object_list[0];
                for (auto const &s3_object : object_list)
                {
                    if (flags){
                        writeLog(L_INFO, "====> 文件下载终止");
                        return false;
                    }
                    /// 设置桶名称
                    const Aws::String bucket_name = stringK;
                    /// 设置待下载的文件名称
                    const Aws::String object_name = s3_object.GetKey();
                    /// 设置保存文件路径及文件名称
                    std::string NewPath;
                    size_t indexNew = object_name.find_last_of('/');
                    NewPath = object_name.substr(indexNew + 1, object_name.length());
                    const Aws::String SavePath = imagePath + NewPath;
                    size_t indexK = downPath.find_last_of('/');
                    std::string newPathEs = downPath.substr(0, indexK);
                    std::string strK;
                    strK = imagePath + "/" + NewPath;
                    const Aws::String SavePaths = strK;

                    //筛选数据
                    std::string fileName;
                    size_t index = object_name.find_last_of('.');
                    fileName = object_name.substr(index + 1, object_name.length());
                    if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){
                        /// 启动下载
                        if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {
                            std::string errorInfo =  object_name + " 下载失败";
                            writeLog(L_ERROR, errorInfo.c_str());
                            continue;
                        }
                    }
                }
                finally_object = object_list[999];
                /// 重新配置相关路径
                objects_request.WithBucket(temps.GetName());   /// 设置桶路径
                objects_request.SetPrefix(finally_object.GetKey());   /// 设置新文件夹路径

                list_objects_outcome = client.ListObjects(objects_request);  /// 判断存储桶及相关连接是否有效
                if (list_objects_outcome.IsSuccess()){
                    object_list = list_objects_outcome.GetResult().GetContents();
                }else{
                    writeLog(L_ERROR, "====> 再次连接失败");
                }
            }
            for (auto const &s3_object : object_list)
            {
                if (flags){
                    writeLog(L_INFO, "====> 文件下载终止");
                    return false;
                }
                /// 设置桶名称
                const Aws::String bucket_name = stringK;
                /// 设置待下载的文件名称
                const Aws::String object_name = s3_object.GetKey();
                /// 设置保存文件路径及文件名称
                std::string NewPath;
                size_t indexNew = object_name.find_last_of('/');
                NewPath = object_name.substr(indexNew + 1, object_name.length());
                const Aws::String SavePath = imagePath + NewPath;
                size_t indexK = downPath.find_last_of('/');
                std::string newPathEs = downPath.substr(0, indexK);

                std::string strK;
                strK = imagePath + "/" + NewPath;
                const Aws::String SavePaths = strK;

                //筛选数据
                std::string fileName;
                size_t index = object_name.find_last_of('.');
                fileName = object_name.substr(index + 1, object_name.length());
                if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){
                    /// 启动下载
                    if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {
                        std::string errorInfo =  object_name + " 下载失败";
                        writeLog(L_ERROR, errorInfo.c_str());
                        continue;
                    }
                }
            }
        }else{
            for (auto const &s3_object : object_list)
            {
                if (flags){
                    writeLog(L_INFO, "====> 文件下载终止");
                    return false;
                }
                /// 设置桶名称
                const Aws::String bucket_name = stringK;
                /// 设置待下载的文件名称
                const Aws::String object_name = s3_object.GetKey();
                /// 设置保存文件路径及文件名称
                std::string NewPath;
                size_t indexNew = object_name.find_last_of('/');
                NewPath = object_name.substr(indexNew + 1, object_name.length());
                const Aws::String SavePath = imagePath +  NewPath;
                size_t indexK = downPath.find_last_of('/');
                std::string newPathEs = downPath.substr(0, indexK);
                std::string strK;
                strK = imagePath + "/" + NewPath;
                const Aws::String SavePaths = strK;

                //筛选数据
                std::string fileName;
                size_t index = object_name.find_last_of('.');
                fileName = object_name.substr(index + 1, object_name.length());
                if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){
                    /// 启动下载
                    if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {
                        std::string errorInfo =  object_name + " 下载失败";
                        writeLog(L_ERROR, errorInfo.c_str());
                        continue;
                    }
                }

            }
        }

    }
    writeLog(L_INFO, "====> 文件下载成功");
    return true;
}