ported zsync meta file parsing

Author: antony-jr

CMakeLists.txt

@@ -0,0 +1,20 @@
+cmake_minimum_required(VERSION 3.4)
+project(zsync3)
+
+set(V_MAJOR 0)
+set(V_MINOR 1)
+set(V_PATCH 0)
+set(V_SUFFIX "")
+
+set(zsync3_VERSION ${V_MAJOR}.${V_MINOR}.${V_PATCH}${V_SUFFIX})
+set(zsync3_SOVERSION ${V_MAJOR}.${V_MINOR})
+
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+set(CMAKE_CXX_FLAGS "-Wall -Wextra")
+set(CMAKE_CXX_FLAGS_DEBUG "-g")
+set(CMAKE_CXX_FLAGS_RELEASE "-O3")
+
+include_directories(.)
+add_subdirectory(src)
+add_subdirectory(tools)

Zsync3

@@ -0,0 +1,2 @@
+//// Proxy file
+#include "include/client.hpp"

include/client.hpp

@@ -0,0 +1,23 @@
+#ifndef ZSYNC3_CLIENT_HPP_INCLUDED
+#define ZSYNC3_CLIENT_HPP_INCLUDED
+#include <memory>
+#include <string>
+
+namespace Zsync3 {
+class ClientPrivate;
+class Client {
+    std::unique_ptr<ClientPrivate> d;
+
+  public:
+    Client();
+    Client(const std::string&);
+    Client(const Client&);
+    ~Client();
+
+    bool SetMetaFile(const std::string&);
+
+    bool SubmitSeedFile(const std::string&);
+};
+}
+
+#endif // ZSYNC3_CLIENT_HPP_INCLUDED

include/client_p.hpp

@@ -0,0 +1,115 @@
+#ifndef ZSYNC3_CLIENT_PRIVATE_HPP_INCLUDED
+#define ZSYNC3_CLIENT_PRIVATE_HPP_INCLUDED
+#include <cstdint>
+#include <cstddef>
+#include <boost/filesystem.hpp>
+#include <openssl/sha.h>
+#include <openssl/md4.h>
+#include <fstream>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace Zsync3 {
+class ClientPrivate {
+    /// Constants
+    const int kBitHashBits = 3;
+    /// ---
+
+    class RollingChecksum {
+        uint16_t a,b;
+      public:
+        RollingChecksum();
+        //// Special Constructor which expects the given variables as
+        //// A anb B which is in network endian order.
+        RollingChecksum(uint16_t, uint16_t);
+        RollingChecksum(const uint8_t*, std::size_t);
+        RollingChecksum(const RollingChecksum&);
+        uint16_t GetA();
+        uint16_t GetB();
+        void update(uint8_t,uint8_t,int32_t);
+
+        void operator = (const RollingChecksum&);
+        bool operator == (const RollingChecksum&);
+        bool operator != (const RollingChecksum&);
+    };
+
+    struct HashEntry {
+        int64_t block_id = 0;
+        RollingChecksum rsum;
+        std::string md4;
+
+        //// Next hash entry with the same rsum
+        HashEntry *next = nullptr;
+    };
+
+    int64_t num_blocks;
+    int blocksize = 0,
+        blockshift = 0; /// log2(blocksize)
+    int64_t num_bytes_written = 0,
+            context = 0,    /* precalculated blocksize * seq_matches */
+            num_weak_checksum_bytes = 0,
+            num_strong_checksum_bytes = 0, /* no. of bytes available for the strong checksum. */
+            num_seq_matches = 0,
+            target_file_length = 0;
+
+    int64_t bit_hash_mask = 0;
+    int64_t hash_mask = 0;
+    uint16_t weak_checksum_mask = 0; /* This will be applied to the first 16 bits of the weak checksum. */
+
+    //// Version string.
+    std::string zs_version;
+
+    //// Date Time string.
+    std::string mtime;
+
+    //// Has all block's hashes.
+    std::unique_ptr<std::vector<HashEntry>> block_hashes;
+
+    //// Has the mapping of rsum to specific HashEntry present in the
+    //// block_hashes.
+    //// The HashEntry is a chain which is a collection of same rsum
+    //// mapping.
+    std::unique_ptr<std::vector<HashEntry*>> rsum_mapping;
+
+    /// This bit hash as 1 bit per rsum value. If a rsum value exists in the
+    /// target file then we will have the relevant bit set to 1 if not
+    /// 0. So this provides us a fast negative lookup.
+    std::unique_ptr<std::vector<uint8_t>> bit_hashes;
+
+    std::unique_ptr<MD4_CTX> md4_ctx;
+    std::unique_ptr<SHA_CTX> sha1_ctx;
+    std::unique_ptr<std::fstream> temp_fs;
+
+    std::string target_file_sha1;
+    std::string target_file_url;
+    std::string target_filename;
+    boost::filesystem::path temp_filename;
+
+  public:
+    ClientPrivate();
+    ~ClientPrivate();
+
+    //// Read a Zsync Meta file and create all neccessary
+    //// data structures to be constructed.
+    bool SetMetaFile(const std::string&);
+
+    //// Read a file and write useable blocks
+    //// to temporary target file.
+    bool SubmitSeedFile(const std::string&);
+  private:
+
+    //// The mapping function for the rsum hashes.
+    //// This function returns the index in where the HashEntry
+    //// resides in the rsum_hashes.
+    uint32_t RsumHash(HashEntry&);
+
+    //// Builds the mapping of rsum to hash entries
+    bool BuildRsumHashTable();
+
+    //// Submit a fragment of data to be zsynced.
+    int64_t SubmitSourceData(const char *, size_t, int64_t);
+};
+}
+
+#endif

src/CMakeLists.txt

@@ -0,0 +1,17 @@
+cmake_minimum_required(VERSION 3.4)
+
+set(source)
+list(APPEND source
+   client_p.cpp
+   client.cpp
+   rollingchecksum.cpp)
+
+if(BUILD_SHARED_LIB)
+	add_library(zsync3 SHARED ${source})
+endif()
+
+if(NOT BUILD_SHARED_LIB)
+	add_library(zsync3 ${source})
+endif()
+
+target_link_libraries(zsync3 crypto pthread)

src/client.cpp

@@ -0,0 +1,27 @@
+#include "../include/client.hpp"
+#include "../include/client_p.hpp"
+
+using namespace Zsync3;
+
+Client::Client() {
+    d.reset(new ClientPrivate);
+}
+
+Client::Client(const Client &other) {
+    this->d.reset(other.d.get());
+}
+
+Client::Client(const std::string &meta_file_path)
+    : Client() {
+    SetMetaFile(meta_file_path);
+}
+
+Client::~Client() { }
+
+bool Client::SetMetaFile(const std::string &path) {
+    return d->SetMetaFile(path);
+}
+
+bool Client::SubmitSeedFile(const std::string &path) {
+    return d->SubmitSeedFile(path);
+}