ms_cdb_sharding.hpp

/*
##############################################################################
# file: ms_cdb_sharding.hpp                                                  #
# 'msparser' toolkit                                                         #
# Sharding of the cdb files based on the content and the size                #
#                                                                            #
##############################################################################
# COPYRIGHT NOTICE                                                           #
# Copyright 1998-2021 Matrix Science Limited  All Rights Reserved.           #
#                                                                            #
##############################################################################
#     $Author: francoisr@matrixscience.com $
#       $Date: 2022-02-18 16:21:35 +0000 $
#   $Revision: b37898d7f040eb50775669c2cfd1f9c88ef4d2da | MSPARSER_REL_3_0_0-2024-09-24-0-g93ebaeb4f4 $
##############################################################################
*/
 
#ifndef MS_CDB_SHARDING_HPP
#define MS_CDB_SHARDING_HPP
 
#ifndef SWIG
 
using namespace matrix_science;
 
namespace msparser_internal {
 
    class CdbPreparingHelper
    {
    public:
        static bool moveFileSafely(const std::string & sourceFile, const std::string & targetFile, std::string & strError);
 
        static bool archiveFileToDirectory(const std::string & archiveDir, const std::string & sourceFile, const std::string & targetFile, std::string & strError);
 
        static bool moveAllFilesInDirectory(const std::string & sourceDir, const std::string & targetDir, std::string & strError, const ms_taxonomyrules * taxRules = nullptr);
    };
 
    class CdbShard : public ms_errors
    {
    public:
 
        CdbShard(const std::string & baseFileName, const char letter, const unsigned numCbdFiles, const bool keepFilesOpen, const bool splitShards);
 
        ~CdbShard();
 
        bool isGood() const { return success_; };
 
        void prepareFiles();
 
        void finaliseFiles();
 
        std::string openFiles();
 
        void closeFiles();
 
        void retargetFiles(const std::string & targetBaseFileName, const std::string & compressionDir, const std::string & targetDir);
 
        char getLetter() const { return letter_; };
 
        int getNumCdb() const { return static_cast<int>(vecCdbFiles_.size()); };
 
        int getIntValue(const std::string & key) const;
 
        bool setIntValue(const std::string & key, const int intValue);
 
    private:
        std::string getCdbFileName(char c, int num) const;
 
        std::string baseFileName_;
        char letter_;
        bool success_;
        UINT64 currentCdbSize_;
        unsigned currentCdbFileIdx_;
        unsigned numCdbFiles_; //only relevant when reading if splitShards is true
 
        //performance tuning
        bool keepFilesOpen_;
        bool splitShards_;
 
        std::vector<matrix_science::ms_tinycdb *> vecCdbFiles_;
    };
 
    class CdbSharding : public ms_errors
    {
    public:
 
        CdbSharding(const std::string & baseFileName, const bool keepFilesOpen, const bool splitShards, const std::string preparingDir = ".");
 
        ~CdbSharding();
 
        bool isGood() const;
 
        bool mustCreate() { return mustCreate_; }
 
        bool prepareShards();
 
        std::string getWorkingBaseFileName();
 
        bool finaliseShards();
 
        bool openShards();
 
        void closeShards();
 
        int getIntValue(const std::string & accession) const;
 
        bool setIntValue(const std::string & accession, const int intValue);
 
    private:
        bool deletePreviousFiles();
        bool isNumberOrLetter(const char c) const;
        CdbShard * getShardForAccession(const std::string & accession) const;
 
        std::string baseFileName_;
        bool success_;
 
        bool mustCreate_;
 
        //performance tuning
        bool keepFilesOpen_;
        bool splitShards_;
 
        matrix_science::ms_tinycdb * mainCdbFile_; //this is used to store how many cdb files each shard has
        std::vector<CdbShard *> vecCharToShard_;
        std::vector<CdbShard *> vecShards_;
 
        std::string preparingDir_;
        std::string sourceDir_;
        std::string baseFileName_withoutPath_;
        bool differentCompressionDir_;
        std::string working_baseFileName_;
    };
 
} // namespace msparser_internal
 
#endif // SWIG
 
#endif // MS_CDB_SHARDING_HPP