string_pool.cpp

#include <iostream>
#include <fstream>
#include <iomanip>
#include <unordered_map>
#include <sparsehash/dense_hash_map>

#include <Common/Stopwatch.h>

#include <Common/Arena.h>

#include <IO/ReadBufferFromFileDescriptor.h>
#include <IO/ReadHelpers.h>

//#define DBMS_HASH_MAP_COUNT_COLLISIONS
#include <Common/HashTable/HashMap.h>
#include <Examples/clickhouse_examples.h>

int mainEntryExampleStringPool(int argc, char ** argv)
{
    if (argc < 2)
    {
        std::cerr << "Usage: program n\n";
        return 1;
    }

    std::cerr << std::fixed << std::setprecision(3);
    std::ofstream devnull("/dev/null");

    DB::ReadBufferFromFileDescriptor in(STDIN_FILENO);
    size_t n = std::stol(argv[1]);
    size_t elems_show = 1;

    using Vec = std::vector<std::string>; // STYLE_CHECK_ALLOW_STD_CONTAINERS
    using Set = std::unordered_map<std::string, int>; // STYLE_CHECK_ALLOW_STD_CONTAINERS
    using RefsSet = std::unordered_map<std::string_view, int, StringViewHash>; // STYLE_CHECK_ALLOW_STD_CONTAINERS
    using DenseSet = ::google::dense_hash_map<std::string, int>;
    using RefsDenseSet = ::google::dense_hash_map<std::string_view, int, StringViewHash>;
    using RefsHashMap = HashMap<std::string_view, int, StringViewHash>;
    Vec vec;

    vec.reserve(n);

    {
        Stopwatch watch;

        std::string s;
        for (size_t i = 0; i < n && !in.eof(); ++i)
        {
            DB::readEscapedString(s, in);
            DB::assertChar('\n', in);
            vec.push_back(s);
        }

        std::cerr << "Read and inserted into vector in " << watch.elapsedSeconds() << " sec, "
            << static_cast<double>(vec.size()) / watch.elapsedSeconds() << " rows/sec., "
            << static_cast<double>(in.count()) / watch.elapsedSeconds() / 1000000 << " MB/sec."
            << std::endl;
    }

    {
        DB::Arena pool;
        Stopwatch watch;
        const char * res = nullptr;

        for (Vec::iterator it = vec.begin(); it != vec.end(); ++it)
        {
            const char * tmp = pool.insert(it->data(), it->size());
            if (it == vec.begin())
                res = tmp;
        }

        std::cerr << "Inserted into pool in " << watch.elapsedSeconds() << " sec, "
            << static_cast<double>(vec.size()) / watch.elapsedSeconds() << " rows/sec., "
            << static_cast<double>(in.count()) / watch.elapsedSeconds() / 1000000 << " MB/sec."
            << std::endl;

        devnull.write(res, 100);
        devnull << std::endl;
    }

    {
        Set set;
        Stopwatch watch;

        for (const auto & elem : vec)
            set[elem] = 0;

        std::cerr << "Inserted into std::unordered_map in " << watch.elapsedSeconds() << " sec, "
            << static_cast<double>(vec.size()) / watch.elapsedSeconds() << " rows/sec., "
            << static_cast<double>(in.count()) / watch.elapsedSeconds() / 1000000 << " MB/sec."
            << std::endl;

        size_t i = 0;
        for (Set::const_iterator it = set.begin(); i < elems_show && it != set.end(); ++it, ++i)
        {
            devnull << it->first;
            devnull << std::endl;
        }
    }

    {
        RefsSet set;
        Stopwatch watch;

        for (const auto & elem : vec)
            set[std::string_view(elem)] = 0;

        std::cerr << "Inserted refs into std::unordered_map in " << watch.elapsedSeconds() << " sec, "
            << static_cast<double>(vec.size()) / watch.elapsedSeconds() << " rows/sec., "
            << static_cast<double>(in.count()) / watch.elapsedSeconds() / 1000000 << " MB/sec."
            << std::endl;

        size_t i = 0;
        for (RefsSet::const_iterator it = set.begin(); i < elems_show && it != set.end(); ++it, ++i)
        {
            devnull.write(it->first.data(), it->first.size());
            devnull << std::endl;
        }
    }

    {
        DB::Arena pool;
        RefsSet set;
        Stopwatch watch;

        for (const auto & elem : vec)
            set[std::string_view(pool.insert(elem.data(), elem.size()), elem.size())] = 0;

        std::cerr << "Inserted into pool and refs into std::unordered_map in " << watch.elapsedSeconds() << " sec, "
            << static_cast<double>(vec.size()) / watch.elapsedSeconds() << " rows/sec., "
            << static_cast<double>(in.count()) / watch.elapsedSeconds() / 1000000 << " MB/sec."
            << std::endl;

        size_t i = 0;
        for (RefsSet::const_iterator it = set.begin(); i < elems_show && it != set.end(); ++it, ++i)
        {
            devnull.write(it->first.data(), it->first.size());
            devnull << std::endl;
        }
    }

    {
        DenseSet set;
        set.set_empty_key(DenseSet::key_type());
        Stopwatch watch;

        for (const auto & elem : vec)
            set[elem] = 0;

        std::cerr << "Inserted into google::dense_hash_map in " << watch.elapsedSeconds() << " sec, "
            << static_cast<double>(vec.size()) / watch.elapsedSeconds() << " rows/sec., "
            << static_cast<double>(in.count()) / watch.elapsedSeconds() / 1000000 << " MB/sec."
            << std::endl;

        size_t i = 0;
        for (DenseSet::const_iterator it = set.begin(); i < elems_show && it != set.end(); ++it, ++i)
        {
            devnull << it->first;
            devnull << std::endl;
        }
    }

    {
        RefsDenseSet set;
        set.set_empty_key(RefsDenseSet::key_type());
        Stopwatch watch;

        for (const auto & elem : vec)
            set[std::string_view(elem.data(), elem.size())] = 0;

        std::cerr << "Inserted refs into google::dense_hash_map in " << watch.elapsedSeconds() << " sec, "
            << static_cast<double>(vec.size()) / watch.elapsedSeconds() << " rows/sec., "
            << static_cast<double>(in.count()) / watch.elapsedSeconds() / 1000000 << " MB/sec."
            << std::endl;

        size_t i = 0;
        for (RefsDenseSet::const_iterator it = set.begin(); i < elems_show && it != set.end(); ++it, ++i)
        {
            devnull.write(it->first.data(), it->first.size());
            devnull << std::endl;
        }
    }

    {
        DB::Arena pool;
        RefsDenseSet set;
        set.set_empty_key(RefsDenseSet::key_type());
        Stopwatch watch;

        for (const auto & elem : vec)
            set[std::string_view(pool.insert(elem.data(), elem.size()), elem.size())] = 0;

        std::cerr << "Inserted into pool and refs into google::dense_hash_map in " << watch.elapsedSeconds() << " sec, "
            << static_cast<double>(vec.size()) / watch.elapsedSeconds() << " rows/sec., "
            << static_cast<double>(in.count()) / watch.elapsedSeconds() / 1000000 << " MB/sec."
            << std::endl;

        size_t i = 0;
        for (RefsDenseSet::const_iterator it = set.begin(); i < elems_show && it != set.end(); ++it, ++i)
        {
            devnull.write(it->first.data(), it->first.size());
            devnull << std::endl;
        }
    }

    {
        RefsHashMap set;
        Stopwatch watch;

        for (const auto & elem : vec)
        {
            RefsHashMap::LookupResult inserted_it = {};
            bool inserted = {};
            set.emplace(std::string_view(elem), inserted_it, inserted);
        }

        std::cerr << "Inserted refs into HashMap in " << watch.elapsedSeconds() << " sec, "
            << static_cast<double>(vec.size()) / watch.elapsedSeconds() << " rows/sec., "
            << static_cast<double>(in.count()) / watch.elapsedSeconds() / 1000000 << " MB/sec."
            << std::endl;

        size_t i = 0;
        for (auto it = set.begin(); i < elems_show && it != set.end(); ++it, ++i)
        {
            devnull.write(it->getKey().data(), it->getKey().size());
            devnull << std::endl;
        }

        //std::cerr << set.size() << ", " << set.getCollisions() << std::endl;
    }

    {
        DB::Arena pool;
        RefsHashMap set;
        Stopwatch watch;

        for (const auto & elem : vec)
        {
            RefsHashMap::LookupResult inserted_it = {};
            bool inserted = {};
            set.emplace(std::string_view(pool.insert(elem.data(), elem.size()), elem.size()), inserted_it, inserted);
        }

        std::cerr << "Inserted into pool and refs into HashMap in " << watch.elapsedSeconds() << " sec, "
            << static_cast<double>(vec.size()) / watch.elapsedSeconds() << " rows/sec., "
            << static_cast<double>(in.count()) / watch.elapsedSeconds() / 1000000 << " MB/sec."
            << std::endl;

        size_t i = 0;
        for (auto it = set.begin(); i < elems_show && it != set.end(); ++it, ++i)
        {
            devnull.write(it->getKey().data(), it->getKey().size());
            devnull << std::endl;
        }
    }

    return 0;
}