ClickHouse/src/Formats/ColumnMapping.cpp at master · StackRadiusForks/ClickHouse · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
#include <Columns/IColumn.h>
#include <Common/Exception.h>
#include <Core/Block.h>
#include <Core/CaseAwareBlockNameMap.h>
#include <Formats/ColumnMapping.h>
#include <Formats/FormatSettings.h>
#include <optional>

namespace DB
{

namespace ErrorCodes
{
    extern const int INCORRECT_DATA;
}

namespace
{
template <typename TSearchFunc>
void addColumnsInternal(
    std::vector<String> & names_of_columns,
    ColumnMapping::OptionalIndexes & column_indexes_for_input_fields,
    std::vector<size_t> & not_presented_columns,
    const Names & column_names,
    const FormatSettings & settings,
    size_t size,
    TSearchFunc find)
{
    std::vector<bool> read_columns(size, false);

    for (const auto & name : column_names)
    {
        names_of_columns.push_back(name);

        std::optional<size_t> column_idx = find(name);
        if (!column_idx.has_value())
        {
            if (settings.skip_unknown_fields)
            {
                column_indexes_for_input_fields.push_back(std::nullopt);
                continue;
            }

            throw Exception(
                ErrorCodes::INCORRECT_DATA,
                "Unknown field found in format header: "
                "'{}' at position {}\nSet the 'input_format_skip_unknown_fields' parameter explicitly "
                "to ignore and proceed",
                name,
                column_indexes_for_input_fields.size());
        }

        auto idx = column_idx.value();

        if (read_columns[idx])
            throw Exception(ErrorCodes::INCORRECT_DATA, "Duplicate field found while parsing format header: {}", name);

        read_columns[idx] = true;
        column_indexes_for_input_fields.emplace_back(column_idx);
    }

    for (size_t i = 0; i != read_columns.size(); ++i)
    {
        if (!read_columns[i])
            not_presented_columns.push_back(i);
    }
}
}

void ColumnMapping::setupByHeader(const Block & header)
{
    column_indexes_for_input_fields.resize(header.columns());
    names_of_columns = header.getNames();

    for (size_t i = 0; i < column_indexes_for_input_fields.size(); ++i)
        column_indexes_for_input_fields[i] = i;
}

void ColumnMapping::addColumns(
    const Names & column_names, const CaseAwareBlockNameMap & column_indexes_by_names, const FormatSettings & settings)
{
    addColumnsInternal(
        names_of_columns,
        column_indexes_for_input_fields,
        not_presented_columns,
        column_names,
        settings,
        column_indexes_by_names.size(),
        [&column_indexes_by_names](const std::string & name)
        {
            auto result = column_indexes_by_names.get(name);
            if (result == CaseAwareBlockNameMap::NOT_FOUND)
            {
                return std::optional<size_t>();
            }
            return std::optional<size_t>(result);
        });
}

void ColumnMapping::addColumns(const Names & column_names, const BlockNameMap & column_indexes_by_names, const FormatSettings & settings)
{
    addColumnsInternal(
        names_of_columns,
        column_indexes_for_input_fields,
        not_presented_columns,
        column_names,
        settings,
        column_indexes_by_names.size(),
        [&column_indexes_by_names](const std::string & name)
        {
            auto it = column_indexes_by_names.find(name);
            if (it == column_indexes_by_names.end())
            {
                return std::optional<size_t>();
            }
            return std::optional<size_t>(it->second);
        });
}

void ColumnMapping::insertDefaultsForNotSeenColumns(MutableColumns & columns, std::vector<UInt8> & read_columns)
{
    for (auto index : not_presented_columns)
    {
        columns[index]->insertDefault();
        read_columns[index] = false;
    }
}

}