diff --git a/be/src/index-tools/index_tool.cpp b/be/src/index-tools/index_tool.cpp index f18ecdc362..3f541caf68 100644 --- a/be/src/index-tools/index_tool.cpp +++ b/be/src/index-tools/index_tool.cpp @@ -18,10 +18,11 @@ #include #include #include +#include #include -#include #include +#include #include #include #include @@ -29,15 +30,32 @@ #include #include +#include "io/fs/file_reader.h" +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshadow-field" +#endif +#include "CLucene/analysis/standard95/StandardAnalyzer.h" +#ifdef __clang__ +#pragma clang diagnostic pop +#endif +#include "gutil/strings/strip.h" #include "io/fs/local_file_system.h" #include "olap/rowset/segment_v2/inverted_index/query/conjunction_query.h" -#include "olap/rowset/segment_v2/inverted_index/query/query.h" -#include "olap/rowset/segment_v2/inverted_index_compound_directory.h" #include "olap/rowset/segment_v2/inverted_index_compound_reader.h" +#include "olap/rowset/segment_v2/inverted_index_desc.h" +#include "olap/rowset/segment_v2/inverted_index_file_reader.h" +#include "olap/rowset/segment_v2/inverted_index_file_writer.h" +#include "olap/rowset/segment_v2/inverted_index_fs_directory.h" +#include "olap/tablet_schema.h" using doris::segment_v2::DorisCompoundReader; -using doris::segment_v2::DorisCompoundDirectoryFactory; +using doris::segment_v2::DorisFSDirectoryFactory; +using doris::segment_v2::InvertedIndexFileWriter; +using doris::segment_v2::InvertedIndexDescriptor; +using doris::segment_v2::InvertedIndexFileReader; using doris::io::FileInfo; +using doris::TabletIndex; using namespace doris::segment_v2; using namespace lucene::analysis; using namespace lucene::index; @@ -49,6 +67,7 @@ DEFINE_string(operation, "", "valid operation: show_nested_files,check_terms,ter DEFINE_string(directory, "./", "inverted index file directory"); DEFINE_string(idx_file_name, "", "inverted index file name"); DEFINE_string(idx_file_path, "", "inverted index file path"); +DEFINE_string(data_file_path, "", "inverted index data path"); DEFINE_string(term, "", "inverted index term to query"); DEFINE_string(column_name, "", "inverted index column_name to query"); DEFINE_string(pred_type, "", "inverted index term query predicate, eq/lt/gt/le/ge/match etc."); @@ -64,6 +83,8 @@ std::string get_usage(const std::string& progname) { ss << "./index_tool --operation=term_query --directory=directory " "--idx_file_name=file --print_row_id --term=term --column_name=column_name " "--pred_type=eq/lt/gt/le/ge/match etc\n"; + ss << "./index_tool --operation=write_index_v2 --idx_file_path=path/to/index " + "--data_file_path=data/to/index\n"; return ss.str(); } @@ -130,9 +151,9 @@ void search(lucene::store::Directory* dir, std::string& field, std::string& toke std::vector terms = split(token, '|'); doris::TQueryOptions queryOptions; - ConjunctionQuery query(s, queryOptions); - query.add(field_ws, terms); - query.search(result); + ConjunctionQuery conjunct_query(s, queryOptions); + conjunct_query.add(field_ws, terms); + conjunct_query.search(result); total += result.cardinality(); } else { @@ -203,8 +224,8 @@ int main(int argc, char** argv) { std::unique_ptr reader; try { reader = std::make_unique( - DorisCompoundDirectoryFactory::getDirectory(fs, dir_str.c_str()), - file_str.c_str(), 4096); + DorisFSDirectoryFactory::getDirectory(fs, dir_str.c_str()), file_str.c_str(), + 4096); std::vector files; std::cout << "Nested files for " << file_str << std::endl; std::cout << "==================================" << std::endl; @@ -238,8 +259,8 @@ int main(int argc, char** argv) { std::unique_ptr reader; try { reader = std::make_unique( - DorisCompoundDirectoryFactory::getDirectory(fs, dir_str.c_str()), - file_str.c_str(), 4096); + DorisFSDirectoryFactory::getDirectory(fs, dir_str.c_str()), file_str.c_str(), + 4096); std::cout << "Term statistics for " << file_str << std::endl; std::cout << "==================================" << std::endl; check_terms_stats(reader.get()); @@ -282,7 +303,7 @@ int main(int argc, char** argv) { continue; } reader = std::make_unique( - DorisCompoundDirectoryFactory::getDirectory(fs, file_str.c_str()), + DorisFSDirectoryFactory::getDirectory(fs, file_str.c_str()), file_str.c_str(), 4096); std::cout << "Search " << FLAGS_column_name << ":" << FLAGS_term << " from " << file_str << std::endl; @@ -305,8 +326,8 @@ int main(int argc, char** argv) { std::cerr << "file " << file_path << " not found" << std::endl; return -1; } - auto reader = std::make_unique( - DorisCompoundDirectoryFactory::getDirectory(fs, FLAGS_directory.c_str()), + reader = std::make_unique( + DorisFSDirectoryFactory::getDirectory(fs, FLAGS_directory.c_str()), FLAGS_idx_file_name.c_str(), 4096); std::cout << "Search " << FLAGS_column_name << ":" << FLAGS_term << " from " << FLAGS_idx_file_name << std::endl; @@ -330,10 +351,196 @@ int main(int argc, char** argv) { } return -1; } + } else if (FLAGS_operation == "write_index_v2") { + if (FLAGS_idx_file_path == "") { + std::cout << "no index path flag for check " << std::endl; + return -1; + } + if (FLAGS_data_file_path == "") { + std::cout << "no data file flag for check " << std::endl; + return -1; + } + std::string name = "test"; + std::string file_dir = FLAGS_idx_file_path; + std::string file_name = "test_index_0.dat"; + int64_t index_id = 1; + std::string index_suffix = ""; + doris::TabletIndexPB index_pb; + index_pb.set_index_id(index_id); + index_pb.set_index_suffix_name(index_suffix); + TabletIndex index_meta; + index_meta.init_from_pb(index_pb); + auto index_path = InvertedIndexDescriptor::get_temporary_index_path( + file_dir + "/" + file_name, index_id, index_suffix); + std::vector datas; + { + if (std::filesystem::exists(FLAGS_data_file_path) && + std::filesystem::file_size(FLAGS_data_file_path) == 0) { + std::cerr << "Error: File '" << FLAGS_data_file_path << "' is empty." << std::endl; + return -1; + } else { + std::ifstream ifs; + std::cout << "prepare to load " << FLAGS_data_file_path << std::endl; + + ifs.open(FLAGS_data_file_path); + if (!ifs) { + std::cerr << "Error: Unable to open file '" << FLAGS_data_file_path << "'." + << std::endl; + return -1; + } else { + std::string line; + while (std::getline(ifs, line)) { + datas.emplace_back(line); + } + ifs.close(); + } + } + } + + auto fs = doris::io::global_local_filesystem(); + auto index_file_writer = std::make_unique( + fs, file_dir, file_name, doris::InvertedIndexStorageFormatPB::V2); + auto st = index_file_writer->open(&index_meta); + if (!st.has_value()) { + std::cerr << "InvertedIndexFileWriter init error:" << st.error() << std::endl; + return -1; + } + using T = std::decay_t; + auto dir = std::forward(st).value(); + auto analyzer = _CLNEW lucene::analysis::standard95::StandardAnalyzer(); + // auto analyzer = _CLNEW lucene::analysis::SimpleAnalyzer(); + auto indexwriter = _CLNEW lucene::index::IndexWriter(dir, analyzer, true, true); + indexwriter->setRAMBufferSizeMB(512); + indexwriter->setMaxFieldLength(0x7FFFFFFFL); + indexwriter->setMergeFactor(100000000); + indexwriter->setUseCompoundFile(false); + + auto char_string_reader = _CLNEW lucene::util::SStringReader; + + auto doc = _CLNEW lucene::document::Document(); + auto field_config = (int32_t)(lucene::document::Field::STORE_NO); + field_config |= (int32_t)(lucene::document::Field::INDEX_NONORMS); + field_config |= lucene::document::Field::INDEX_TOKENIZED; + auto field_name = std::wstring(name.begin(), name.end()); + auto field = _CLNEW lucene::document::Field(field_name.c_str(), field_config); + field->setOmitTermFreqAndPositions(false); + doc->add(*field); + + for (int32_t j = 0; j < 1; j++) { + for (auto& str : datas) { + char_string_reader->init(str.data(), str.size(), false); + auto stream = analyzer->reusableTokenStream(field->name(), char_string_reader); + field->setValue(stream); + + // field->setValue(str.data(), str.size()); + + indexwriter->addDocument(doc); + } + } + indexwriter->close(); + + _CLLDELETE(indexwriter); + _CLLDELETE(doc); + _CLLDELETE(analyzer); + _CLLDELETE(char_string_reader); + + auto ret = index_file_writer->close(); + if (!ret.ok()) { + std::cerr << "InvertedIndexFileWriter close error:" << ret.msg() << std::endl; + return -1; + } + } else if (FLAGS_operation == "show_nested_files_v2") { + if (FLAGS_idx_file_path == "") { + std::cout << "no file flag for show " << std::endl; + return -1; + } + std::filesystem::path p(FLAGS_idx_file_path); + auto dir_path = p.parent_path(); + std::string file_str = StripSuffixString(p.filename().string(), ".idx"); + auto fs = doris::io::global_local_filesystem(); + try { + auto index_file_reader = std::make_unique( + fs, dir_path, file_str, doris::InvertedIndexStorageFormatPB::V2); + auto st = index_file_reader->init(4096); + if (!st.ok()) { + std::cerr << "InvertedIndexFileReader init error:" << st.msg() << std::endl; + return -1; + } + std::cout << "Nested files for " << file_str << std::endl; + std::cout << "==================================" << std::endl; + auto dirs = index_file_reader->get_all_directories(); + for (auto& dir : *dirs) { + auto index_id = dir.first.first; + auto index_suffix = dir.first.second; + std::vector files; + doris::TabletIndexPB index_pb; + index_pb.set_index_id(index_id); + index_pb.set_index_suffix_name(index_suffix); + TabletIndex index_meta; + index_meta.init_from_pb(index_pb); + std::cout << "index_id:" << index_id << " index_suffix:" << index_suffix + << std::endl; + + CLuceneError err; + auto ret = index_file_reader->open(&index_meta); + if (!ret.has_value()) { + std::cerr << "InvertedIndexFileReader open error:" << ret.error() << std::endl; + return -1; + } + using T = std::decay_t; + auto reader = std::forward(ret).value(); + reader->list(&files); + for (auto& file : files) { + std::cout << file << std::endl; + } + } + } catch (CLuceneError& err) { + std::cerr << "error occurred when show files: " << err.what() << std::endl; + } + } else if (FLAGS_operation == "check_terms_stats_v2") { + if (FLAGS_idx_file_path == "") { + std::cout << "no file flag for check " << std::endl; + return -1; + } + std::filesystem::path p(FLAGS_idx_file_path); + std::string dir_path = p.parent_path(); + std::string file_str = StripSuffixString(p.filename().string(), ".idx"); + auto fs = doris::io::global_local_filesystem(); + try { + auto index_file_reader = std::make_unique( + fs, dir_path, file_str, doris::InvertedIndexStorageFormatPB::V2); + auto st = index_file_reader->init(4096); + if (!st.ok()) { + std::cerr << "InvertedIndexFileReader init error:" << st.msg() << std::endl; + return -1; + } + std::vector files; + int64_t index_id = 1; + std::string index_suffix = ""; + doris::TabletIndexPB index_pb; + index_pb.set_index_id(index_id); + index_pb.set_index_suffix_name(index_suffix); + TabletIndex index_meta; + index_meta.init_from_pb(index_pb); + auto ret = index_file_reader->open(&index_meta); + if (!ret.has_value()) { + std::cerr << "InvertedIndexFileReader open error:" << ret.error() << std::endl; + return -1; + } + using T = std::decay_t; + auto reader = std::forward(ret).value(); + index_file_reader->debug_file_entries(); + std::cout << "Term statistics for " << file_str << std::endl; + std::cout << "==================================" << std::endl; + check_terms_stats(reader.get()); + reader->close(); + } catch (CLuceneError& err) { + std::cerr << "error occurred when check_terms_stats: " << err.what() << std::endl; + } } else { std::cerr << "invalid operation: " << FLAGS_operation << "\n" << usage << std::endl; return -1; } gflags::ShutDownCommandLineFlags(); return 0; -} \ No newline at end of file +} diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 0ccb1c53aa..c4dd3cf31c 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -47,7 +47,9 @@ #include "olap/rowset/rowset_writer.h" #include "olap/rowset/rowset_writer_context.h" #include "olap/rowset/segment_v2/inverted_index_compaction.h" -#include "olap/rowset/segment_v2/inverted_index_compound_directory.h" +#include "olap/rowset/segment_v2/inverted_index_file_reader.h" +#include "olap/rowset/segment_v2/inverted_index_file_writer.h" +#include "olap/rowset/segment_v2/inverted_index_fs_directory.h" #include "olap/storage_engine.h" #include "olap/storage_policy.h" #include "olap/tablet.h" @@ -473,63 +475,118 @@ Status Compaction::do_compaction_impl(int64_t permits) { const auto& fs = _output_rowset->rowset_meta()->fs(); const auto& tablet_path = _tablet->tablet_path(); + // src index dirs + // format: rowsetId_segmentId + std::vector> inverted_index_file_readers( + src_segment_num); + for (const auto& m : src_seg_to_id_map) { + std::pair p = m.first; + auto segment_file_name = + p.first.to_string() + "_" + std::to_string(p.second) + ".dat"; + auto inverted_index_file_reader = std::make_unique( + fs, tablet_path, segment_file_name, + _cur_tablet_schema->get_inverted_index_storage_format()); + bool open_idx_file_cache = false; + auto st = inverted_index_file_reader->init(config::inverted_index_read_buffer_size, + open_idx_file_cache); + if (!st.ok()) { + LOG(ERROR) << "init inverted index " + << InvertedIndexDescriptor::get_index_file_name(segment_file_name) + << " failed in compaction when init inverted index file reader"; + return st; + } + inverted_index_file_readers[m.second] = std::move(inverted_index_file_reader); + } + + // dest index files + // format: rowsetId_segmentId + std::vector> inverted_index_file_writers( + dest_segment_num); + for (int i = 0; i < dest_segment_num; ++i) { + auto prefix = dest_rowset_id.to_string() + "_" + std::to_string(i) + ".dat"; + auto inverted_index_file_reader = std::make_unique( + fs, tablet_path, prefix, + _cur_tablet_schema->get_inverted_index_storage_format()); + bool open_idx_file_cache = false; + auto st = inverted_index_file_reader->init(config::inverted_index_read_buffer_size, + open_idx_file_cache); + if (st.ok()) { + auto index_not_need_to_compact = + DORIS_TRY(inverted_index_file_reader->get_all_directories()); + auto inverted_index_file_writer = std::make_unique( + fs, tablet_path, prefix, + _cur_tablet_schema->get_inverted_index_storage_format()); + RETURN_NOT_OK_STATUS_WITH_WARN( + inverted_index_file_writer->initialize(index_not_need_to_compact), + "failed to initialize inverted_index_file_writer for " + + inverted_index_file_writer->get_index_file_name()); + inverted_index_file_writers[i] = std::move(inverted_index_file_writer); + } else if (st.is()) { + auto inverted_index_file_writer = std::make_unique( + fs, tablet_path, prefix, + _cur_tablet_schema->get_inverted_index_storage_format()); + inverted_index_file_writers[i] = std::move(inverted_index_file_writer); + } else { + LOG(ERROR) << "init inverted index " + << InvertedIndexDescriptor::get_index_file_name(prefix) + << " failed in compaction when create inverted index file writer"; + return st; + } + } + // we choose the first destination segment name as the temporary index writer path // Used to distinguish between different index compaction - auto index_writer_path = tablet_path + "/" + dest_index_files[0]; + auto index_tmp_path = tablet_path + "/" + dest_rowset_id.to_string() + "_" + "tmp"; LOG(INFO) << "start index compaction" << ". tablet=" << _tablet->tablet_id() << ", source index size=" << src_segment_num << ", destination index size=" << dest_segment_num << "."; - Status status = Status::OK(); - std::for_each( - ctx.skip_inverted_index.cbegin(), ctx.skip_inverted_index.cend(), - [&src_segment_num, &dest_segment_num, &index_writer_path, &src_index_files, - &dest_index_files, &fs, &tablet_path, &trans_vec, &dest_segment_num_rows, - &status, this](int32_t column_uniq_id) { - auto index_id = _cur_tablet_schema->get_inverted_index(column_uniq_id, "") - ->index_id(); - try { - auto st = compact_column(index_id, src_segment_num, dest_segment_num, - src_index_files, dest_index_files, fs, - index_writer_path, tablet_path, trans_vec, - dest_segment_num_rows); - if (!st.ok()) { - LOG(WARNING) << "failed to do index compaction" - << ". tablet=" << _tablet->tablet_id() - << ". column uniq id=" << column_uniq_id - << ". index_id=" << index_id; - for (auto& rowset : _input_rowsets) { - rowset->set_skip_index_compaction(column_uniq_id); - LOG(INFO) << "mark skipping inverted index compaction next time" - << ". tablet=" << _tablet->tablet_id() - << ", rowset=" << rowset->rowset_id() - << ", column uniq id=" << column_uniq_id - << ", index_id=" << index_id; - } - status = Status::Error( - st.msg()); - } - } catch (CLuceneError& e) { - LOG(WARNING) << "failed to do index compaction" - << ". tablet=" << _tablet->tablet_id() - << ", column uniq id=" << column_uniq_id - << ", index_id=" << index_id; - for (auto& rowset : _input_rowsets) { - rowset->set_skip_index_compaction(column_uniq_id); - LOG(INFO) << "mark skipping inverted index compaction next time" - << ". tablet=" << _tablet->tablet_id() - << ", rowset=" << rowset->rowset_id() - << ", column uniq id=" << column_uniq_id - << ", index_id=" << index_id; - } - status = Status::Error( - e.what()); - } - }); - // check index compaction status. If status is not ok, we should return error and end this compaction round. - if (!status.ok()) { - return status; + auto error_handler = [this](int64_t index_id, int64_t column_uniq_id) { + LOG(WARNING) << "failed to do index compaction" + << ". tablet=" << _tablet->tablet_id() + << ". column uniq id=" << column_uniq_id << ". index_id=" << index_id; + for (auto& rowset : _input_rowsets) { + rowset->set_skip_index_compaction(column_uniq_id); + LOG(INFO) << "mark skipping inverted index compaction next time" + << ". tablet=" << _tablet->tablet_id() + << ", rowset=" << rowset->rowset_id() + << ", column uniq id=" << column_uniq_id << ", index_id=" << index_id; + } + }; + + for (auto&& column_uniq_id : ctx.skip_inverted_index) { + auto col = _cur_tablet_schema->column_by_uid(column_uniq_id); + const auto* index_meta = _cur_tablet_schema->get_inverted_index(col); + std::vector dest_index_dirs(dest_segment_num); + std::vector src_index_dirs(src_segment_num); + try { + for (int src_segment_id = 0; src_segment_id < src_segment_num; + src_segment_id++) { + auto src_dir = DORIS_TRY( + inverted_index_file_readers[src_segment_id]->open(index_meta)); + src_index_dirs[src_segment_id] = src_dir.release(); + } + for (int dest_segment_id = 0; dest_segment_id < dest_segment_num; + dest_segment_id++) { + auto* dest_dir = DORIS_TRY( + inverted_index_file_writers[dest_segment_id]->open(index_meta)); + dest_index_dirs[dest_segment_id] = dest_dir; + } + auto st = + compact_column(index_meta->index_id(), src_index_dirs, dest_index_dirs, + fs, index_tmp_path, trans_vec, dest_segment_num_rows); + if (!st.ok()) { + error_handler(index_meta->index_id(), column_uniq_id); + return Status::Error(st.msg()); + } + } catch (CLuceneError& e) { + error_handler(index_meta->index_id(), column_uniq_id); + return Status::Error(e.what()); + } + } + for (auto& inverted_index_file_writer : inverted_index_file_writers) { + RETURN_IF_ERROR(inverted_index_file_writer->close()); } LOG(INFO) << "succeed to do index compaction" @@ -636,63 +693,71 @@ Status Compaction::construct_output_rowset_writer(RowsetWriterContext& ctx, bool } for (auto i = 0; i < rowset->num_segments(); i++) { auto segment_file = rowset->segment_file_path(i); - std::string inverted_index_src_file_path = - InvertedIndexDescriptor::get_index_file_name( - segment_file, index_meta->index_id(), - index_meta->get_index_suffix()); - bool exists = false; - if (!fs->exists(inverted_index_src_file_path, &exists).ok()) { - LOG(ERROR) - << inverted_index_src_file_path << " fs->exists error"; + io::Path segment_path(segment_file); + auto inverted_index_file_reader = + std::make_unique( + fs, segment_path.parent_path(), + segment_path.filename(), + _cur_tablet_schema + ->get_inverted_index_storage_format()); + bool open_idx_file_cache = false; + auto st = inverted_index_file_reader->init( + config::inverted_index_read_buffer_size, + open_idx_file_cache); + if (!st.ok()) { + LOG(WARNING) << "init index " + << inverted_index_file_reader->get_index_file_path( + index_meta) + << " error:" << st; return false; } + + bool exists = false; + if (!inverted_index_file_reader + ->index_file_exist(index_meta, &exists) + .ok()) { + LOG(ERROR) << inverted_index_file_reader->get_index_file_path( + index_meta) + << " fs->exists error"; + return false; + } + if (!exists) { LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] column_unique_id[" << col_unique_id << "]," - << inverted_index_src_file_path + << inverted_index_file_reader->get_index_file_path( + index_meta) << " is not exists, will skip index compaction"; return false; } - // check idx file size - int64_t file_size = 0; - if (fs->file_size(inverted_index_src_file_path, &file_size) != - Status::OK()) { - LOG(ERROR) << inverted_index_src_file_path - << " fs->file_size error"; - return false; - } - if (file_size == 0) { - LOG(WARNING) << "tablet[" << _tablet->tablet_id() - << "] column_unique_id[" << col_unique_id << "]," - << inverted_index_src_file_path - << " is empty file, will skip index compaction"; - return false; - } - // check index meta - std::filesystem::path p(inverted_index_src_file_path); - std::string dir_str = p.parent_path().string(); - std::string file_str = p.filename().string(); - lucene::store::Directory* dir = - DorisCompoundDirectoryFactory::getDirectory( - fs, dir_str.c_str()); - DorisCompoundReader reader(dir, file_str.c_str()); + auto result = inverted_index_file_reader->open(index_meta); + if (!result.has_value()) { + LOG(WARNING) << "open index " + << inverted_index_file_reader->get_index_file_path( + index_meta) + << " error:" << result.error(); + return false; + } + auto reader = std::move(result.value()); std::vector files; - reader.list(&files); - reader.close(); + reader->list(&files); + reader->close(); // why is 3? // bkd index will write at least 3 files if (files.size() < 3) { LOG(WARNING) << "tablet[" << _tablet->tablet_id() << "] column_unique_id[" << col_unique_id << "]," - << inverted_index_src_file_path + << inverted_index_file_reader->get_index_file_path( + index_meta) << " is corrupted, will skip index compaction"; return false; } } return true; + return true; }); if (all_have_inverted_index && field_is_slice_type(_cur_tablet_schema->column_by_uid(col_unique_id).type())) { diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index c5f0a09d49..53d68b9d3d 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -272,20 +272,39 @@ void DeltaWriter::_request_slave_tablet_pull_rowset(PNodeInfo node_info) { request->mutable_segments_size()->insert({segment_id, segment_size}); if (!indices_ids.empty()) { - for (auto index_id : indices_ids) { + if (tablet_schema->get_inverted_index_storage_format() != + InvertedIndexStorageFormatPB::V1) { std::string inverted_index_file = InvertedIndexDescriptor::get_index_file_name( - tablet_path + "/" + segment_name.str(), index_id.first, index_id.second); + tablet_path + "/" + segment_name.str()); int64_t size = std::filesystem::file_size(inverted_index_file); PTabletWriteSlaveRequest::IndexSize index_size; - index_size.set_indexid(index_id.first); + // special id for non-V1 format + index_size.set_indexid(0); index_size.set_size(size); - index_size.set_suffix_path(index_id.second); + index_size.set_suffix_path(""); // Fetch the map value for the current segment_id. // If it doesn't exist, this will insert a new default-constructed IndexSizeMapValue auto& index_size_map_value = (*(request->mutable_inverted_indices_size()))[segment_id]; // Add the new index size to the map value. *index_size_map_value.mutable_index_sizes()->Add() = std::move(index_size); + } else { + for (auto index_id : indices_ids) { + std::string inverted_index_file = InvertedIndexDescriptor::get_index_file_name( + tablet_path + "/" + segment_name.str(), index_id.first, + index_id.second); + int64_t size = std::filesystem::file_size(inverted_index_file); + PTabletWriteSlaveRequest::IndexSize index_size; + index_size.set_indexid(index_id.first); + index_size.set_size(size); + index_size.set_suffix_path(index_id.second); + // Fetch the map value for the current segment_id. + // If it doesn't exist, this will insert a new default-constructed IndexSizeMapValue + auto& index_size_map_value = + (*(request->mutable_inverted_indices_size()))[segment_id]; + // Add the new index size to the map value. + *index_size_map_value.mutable_index_sizes()->Add() = std::move(index_size); + } } } } diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 58e495be22..9874f0ce47 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -92,22 +92,34 @@ Status BetaRowset::do_load(bool /*use_cache*/) { return Status::OK(); } -Status BetaRowset::get_inverted_index_size_by_index_id(int64_t index_id, size_t* index_size) { +Status BetaRowset::get_inverted_index_size(size_t* index_size) { auto fs = _rowset_meta->fs(); if (!fs || _schema == nullptr) { return Status::Error("get fs failed"); } - for (int seg_id = 0; seg_id < num_segments(); ++seg_id) { - auto seg_path = segment_file_path(seg_id); - int64_t file_size = 0; - const auto* index = _schema->get_inverted_index_with_index_id(index_id, ""); - if (index == nullptr || index->index_type() != IndexType::INVERTED) { - continue; + if (_schema->get_inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { + auto indices = _schema->indexes(); + for (auto& index : indices) { + for (int seg_id = 0; seg_id < num_segments(); ++seg_id) { + auto seg_path = segment_file_path(seg_id); + int64_t file_size = 0; + + std::string inverted_index_file_path = InvertedIndexDescriptor::get_index_file_name( + seg_path, index.index_id(), index.get_index_suffix()); + RETURN_IF_ERROR(fs->file_size(inverted_index_file_path, &file_size)); + *index_size += file_size; + } + } + } else { + for (int seg_id = 0; seg_id < num_segments(); ++seg_id) { + auto seg_path = segment_file_path(seg_id); + int64_t file_size = 0; + + std::string inverted_index_file_path = + InvertedIndexDescriptor::get_index_file_name(seg_path); + RETURN_IF_ERROR(fs->file_size(inverted_index_file_path, &file_size)); + *index_size += file_size; } - std::string inverted_index_file_path = InvertedIndexDescriptor::get_index_file_name( - seg_path, index_id, index->get_index_suffix()); - RETURN_IF_ERROR(fs->file_size(inverted_index_file_path, &file_size)); - *index_size += file_size; } return Status::OK(); } @@ -191,16 +203,30 @@ Status BetaRowset::remove() { LOG(WARNING) << st.to_string(); success = false; } + if (_schema->get_inverted_index_storage_format() != InvertedIndexStorageFormatPB::V1 && + _schema->has_inverted_index()) { + std::string inverted_index_file = + InvertedIndexDescriptor::get_index_file_name(seg_path); + st = fs->delete_file(inverted_index_file); + if (!st.ok()) { + LOG(WARNING) << st.to_string(); + success = false; + } + } for (auto& column : _schema->columns()) { const TabletIndex* index_meta = _schema->get_inverted_index(*column); if (index_meta) { std::string inverted_index_file = InvertedIndexDescriptor::get_index_file_name( seg_path, index_meta->index_id(), index_meta->get_index_suffix()); - st = fs->delete_file(inverted_index_file); - if (!st.ok()) { - LOG(WARNING) << st.to_string(); - success = false; - } else { + if (_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::V1) { + st = fs->delete_file(inverted_index_file); + if (!st.ok()) { + LOG(WARNING) << st.to_string(); + success = false; + } + } + if (success) { RETURN_IF_ERROR(segment_v2::InvertedIndexSearcherCache::instance()->erase( inverted_index_file)); } @@ -263,44 +289,77 @@ Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id, return status; } linked_success_files.push_back(dst_path); - for (auto& index : _schema->indexes()) { - if (index.index_type() != IndexType::INVERTED) { - continue; - } + DBUG_EXECUTE_IF("fault_inject::BetaRowset::link_files_to::_link_inverted_index_file", { + status = Status::Error("fault_inject link_file error"); + return status; + }); + if (_schema->get_inverted_index_storage_format() != InvertedIndexStorageFormatPB::V1) { + if (_schema->has_inverted_index() && + (without_index_uids == nullptr || without_index_uids->empty())) { + std::string inverted_index_file_src = + InvertedIndexDescriptor::get_index_file_name(src_path); + std::string inverted_index_file_dst = + InvertedIndexDescriptor::get_index_file_name(dst_path); + bool index_dst_path_exist = false; - auto index_id = index.index_id(); - if (without_index_uids != nullptr && without_index_uids->count(index_id)) { - continue; - } - std::string inverted_index_src_file_path = InvertedIndexDescriptor::get_index_file_name( - src_path, index_id, index.get_index_suffix()); - std::string inverted_index_dst_file_path = InvertedIndexDescriptor::get_index_file_name( - dst_path, index_id, index.get_index_suffix()); - bool index_file_exists = true; - RETURN_IF_ERROR(local_fs->exists(inverted_index_src_file_path, &index_file_exists)); - if (index_file_exists) { - DBUG_EXECUTE_IF( - "fault_inject::BetaRowset::link_files_to::_link_inverted_index_file", { - status = Status::Error( - "fault_inject link_file error from={}, to={}", - inverted_index_src_file_path, inverted_index_dst_file_path); - return status; - }); - if (!local_fs->link_file(inverted_index_src_file_path, inverted_index_dst_file_path) - .ok()) { - status = Status::Error( - "fail to create hard link. from={}, to={}, errno={}", - inverted_index_src_file_path, inverted_index_dst_file_path, - Errno::no()); + if (!fs->exists(inverted_index_file_dst, &index_dst_path_exist).ok() || + index_dst_path_exist) { + status = Status::Error( + "failed to create hard link, file already exist: {}", + inverted_index_file_dst); return status; } - linked_success_files.push_back(inverted_index_dst_file_path); - LOG(INFO) << "success to create hard link. from=" << inverted_index_src_file_path - << ", " - << "to=" << inverted_index_dst_file_path; - } else { - LOG(WARNING) << "skip create hard link to not existed index file=" - << inverted_index_src_file_path; + if (!local_fs->link_file(inverted_index_file_src, inverted_index_file_dst).ok()) { + status = Status::Error( + "fail to create hard link. from={}, to={}, errno={}", + inverted_index_file_src, inverted_index_file_dst, Errno::no()); + return status; + } + linked_success_files.push_back(inverted_index_file_dst); + } + } else { + for (const auto& index : _schema->indexes()) { + if (index.index_type() != IndexType::INVERTED) { + continue; + } + + auto index_id = index.index_id(); + if (without_index_uids != nullptr && without_index_uids->count(index_id)) { + continue; + } + std::string inverted_index_src_file_path = + InvertedIndexDescriptor::get_index_file_name(src_path, index_id, + index.get_index_suffix()); + std::string inverted_index_dst_file_path = + InvertedIndexDescriptor::get_index_file_name(dst_path, index_id, + index.get_index_suffix()); + bool index_file_exists = true; + RETURN_IF_ERROR(local_fs->exists(inverted_index_src_file_path, &index_file_exists)); + if (index_file_exists) { + DBUG_EXECUTE_IF( + "fault_inject::BetaRowset::link_files_to::_link_inverted_index_file", { + status = Status::Error( + "fault_inject link_file error from={}, to={}", + inverted_index_src_file_path, inverted_index_dst_file_path); + return status; + }); + if (!local_fs->link_file(inverted_index_src_file_path, + inverted_index_dst_file_path) + .ok()) { + status = Status::Error( + "fail to create hard link. from={}, to={}, errno={}", + inverted_index_src_file_path, inverted_index_dst_file_path, + Errno::no()); + return status; + } + linked_success_files.push_back(inverted_index_dst_file_path); + LOG(INFO) << "success to create hard link. from=" + << inverted_index_src_file_path << ", " + << "to=" << inverted_index_dst_file_path; + } else { + LOG(WARNING) << "skip create hard link to not existed index file=" + << inverted_index_src_file_path; + } } } } @@ -318,20 +377,36 @@ Status BetaRowset::copy_files_to(const std::string& dir, const RowsetId& new_row } auto src_path = segment_file_path(i); RETURN_IF_ERROR(io::global_local_filesystem()->copy_path(src_path, dst_path)); - for (auto& column : _schema->columns()) { - // if (column.has_inverted_index()) { - const TabletIndex* index_meta = _schema->get_inverted_index(*column); - if (index_meta) { - std::string inverted_index_src_file_path = - InvertedIndexDescriptor::get_index_file_name( - src_path, index_meta->index_id(), index_meta->get_index_suffix()); - std::string inverted_index_dst_file_path = - InvertedIndexDescriptor::get_index_file_name( - dst_path, index_meta->index_id(), index_meta->get_index_suffix()); - RETURN_IF_ERROR(io::global_local_filesystem()->copy_path( - inverted_index_src_file_path, inverted_index_dst_file_path)); - LOG(INFO) << "success to copy file. from=" << inverted_index_src_file_path << ", " - << "to=" << inverted_index_dst_file_path; + if (_schema->get_inverted_index_storage_format() != InvertedIndexStorageFormatPB::V1) { + if (_schema->has_inverted_index()) { + std::string inverted_index_dst_file = + InvertedIndexDescriptor::get_index_file_name(dst_path); + std::string inverted_index_src_file = + InvertedIndexDescriptor::get_index_file_name(src_path); + RETURN_IF_ERROR(io::global_local_filesystem()->copy_path(inverted_index_src_file, + inverted_index_dst_file)); + LOG(INFO) << "success to copy file. from=" << inverted_index_src_file << ", " + << "to=" << inverted_index_dst_file; + } + } else { + for (auto& column : _schema->columns()) { + // if (column.has_inverted_index()) { + const TabletIndex* index_meta = _schema->get_inverted_index(*column); + if (index_meta) { + std::string inverted_index_src_file_path = + InvertedIndexDescriptor::get_index_file_name( + src_path, index_meta->index_id(), + index_meta->get_index_suffix()); + std::string inverted_index_dst_file_path = + InvertedIndexDescriptor::get_index_file_name( + dst_path, index_meta->index_id(), + index_meta->get_index_suffix()); + RETURN_IF_ERROR(io::global_local_filesystem()->copy_path( + inverted_index_src_file_path, inverted_index_dst_file_path)); + LOG(INFO) << "success to copy file. from=" << inverted_index_src_file_path + << ", " + << "to=" << inverted_index_dst_file_path; + } } } } @@ -351,22 +426,33 @@ Status BetaRowset::upload_to(io::RemoteFileSystem* dest_fs, const RowsetId& new_ // Note: Here we use relative path for remote. auto remote_seg_path = remote_segment_path(_rowset_meta->tablet_id(), new_rowset_id, i); auto local_seg_path = segment_file_path(i); - dest_paths.push_back(remote_seg_path); - local_paths.push_back(local_seg_path); - for (auto& column : _schema->columns()) { - // if (column.has_inverted_index()) { - const TabletIndex* index_meta = _schema->get_inverted_index(*column); - if (index_meta) { + dest_paths.emplace_back(remote_seg_path); + local_paths.emplace_back(local_seg_path); + if (_schema->get_inverted_index_storage_format() != InvertedIndexStorageFormatPB::V1) { + if (_schema->has_inverted_index()) { std::string remote_inverted_index_file = - InvertedIndexDescriptor::get_index_file_name( - remote_seg_path, index_meta->index_id(), - index_meta->get_index_suffix()); + InvertedIndexDescriptor::get_index_file_name(remote_seg_path); std::string local_inverted_index_file = - InvertedIndexDescriptor::get_index_file_name( - local_seg_path, index_meta->index_id(), - index_meta->get_index_suffix()); - dest_paths.push_back(remote_inverted_index_file); - local_paths.push_back(local_inverted_index_file); + InvertedIndexDescriptor::get_index_file_name(local_seg_path); + dest_paths.emplace_back(remote_inverted_index_file); + local_paths.emplace_back(local_inverted_index_file); + } + } else { + for (auto& column : _schema->columns()) { + // if (column.has_inverted_index()) { + const TabletIndex* index_meta = _schema->get_inverted_index(*column); + if (index_meta) { + std::string remote_inverted_index_file = + InvertedIndexDescriptor::get_index_file_name( + remote_seg_path, index_meta->index_id(), + index_meta->get_index_suffix()); + std::string local_inverted_index_file = + InvertedIndexDescriptor::get_index_file_name( + local_seg_path, index_meta->index_id(), + index_meta->get_index_suffix()); + dest_paths.emplace_back(remote_inverted_index_file); + local_paths.emplace_back(local_inverted_index_file); + } } } } diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h index 9d87eb6bf7..46f16f4d50 100644 --- a/be/src/olap/rowset/beta_rowset.h +++ b/be/src/olap/rowset/beta_rowset.h @@ -94,7 +94,7 @@ public: Status load_segment(int64_t seg_id, segment_v2::SegmentSharedPtr* segment); Status get_segments_size(std::vector* segments_size); - Status get_inverted_index_size_by_index_id(int64_t index_id, size_t* index_size); + Status get_inverted_index_size(size_t* index_size); [[nodiscard]] virtual Status add_to_binlog() override; diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 922aa633ef..4bea55e57f 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -351,6 +351,26 @@ Status BetaRowsetWriter::_rename_compacted_segment_plain(uint64_t seg_id) { Status BetaRowsetWriter::_rename_compacted_indices(int64_t begin, int64_t end, uint64_t seg_id) { int ret; + if (_context.tablet_schema->get_inverted_index_storage_format() != + InvertedIndexStorageFormatPB::V1) { + if (_context.tablet_schema->has_inverted_index()) { + auto src_seg_path = + begin < 0 ? BetaRowset::segment_file_path(_context.rowset_dir, + _context.rowset_id, seg_id) + : BetaRowset::local_segment_path_segcompacted( + _context.rowset_dir, _context.rowset_id, begin, end); + auto dst_seg_path = BetaRowset::segment_file_path( + _context.rowset_dir, _context.rowset_id, _num_segcompacted); + auto src_idx_path = InvertedIndexDescriptor::get_index_file_name(src_seg_path); + auto dst_idx_path = InvertedIndexDescriptor::get_index_file_name(dst_seg_path); + ret = rename(src_idx_path.c_str(), dst_idx_path.c_str()); + if (ret) { + return Status::Error( + "failed to rename {} to {}. ret:{}, errno:{}", src_idx_path, dst_idx_path, + ret, errno); + } + } + } // rename remaining inverted index files for (auto column : _context.tablet_schema->columns()) { if (_context.tablet_schema->has_inverted_index(*column)) { @@ -366,13 +386,16 @@ Status BetaRowsetWriter::_rename_compacted_indices(int64_t begin, int64_t end, u auto dst_idx_path = InvertedIndexDescriptor::inverted_index_file_path( _context.rowset_dir, _context.rowset_id, _num_segcompacted, index_id, index_info->get_index_suffix()); - VLOG_DEBUG << "segcompaction skip this index. rename " << src_idx_path << " to " - << dst_idx_path; - ret = rename(src_idx_path.c_str(), dst_idx_path.c_str()); - if (ret) { - return Status::Error( - "failed to rename {} to {}. ret:{}, errno:{}", src_idx_path, dst_idx_path, - ret, errno); + if (_context.tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::V1) { + VLOG_DEBUG << "segcompaction skip this index. rename " << src_idx_path << " to " + << dst_idx_path; + ret = rename(src_idx_path.c_str(), dst_idx_path.c_str()); + if (ret) { + return Status::Error( + "failed to rename {} to {}. ret:{}, errno:{}", src_idx_path, + dst_idx_path, ret, errno); + } } // Erase the origin index file cache RETURN_IF_ERROR(InvertedIndexSearcherCache::instance()->erase(src_idx_path)); diff --git a/be/src/olap/rowset/segcompaction.cpp b/be/src/olap/rowset/segcompaction.cpp index 402a635da6..f017f53448 100644 --- a/be/src/olap/rowset/segcompaction.cpp +++ b/be/src/olap/rowset/segcompaction.cpp @@ -133,6 +133,14 @@ Status SegcompactionWorker::_delete_original_segments(uint32_t begin, uint32_t e // message when we encounter an error. RETURN_NOT_OK_STATUS_WITH_WARN(fs->delete_file(seg_path), strings::Substitute("Failed to delete file=$0", seg_path)); + if (schema->has_inverted_index() && + schema->get_inverted_index_storage_format() != InvertedIndexStorageFormatPB::V1) { + auto idx_path = InvertedIndexDescriptor::get_index_file_name(seg_path); + VLOG_DEBUG << "segcompaction index. delete file " << idx_path; + RETURN_NOT_OK_STATUS_WITH_WARN( + fs->delete_file(idx_path), + strings::Substitute("Failed to delete file=$0", idx_path)); + } // Delete inverted index files for (auto column : schema->columns()) { if (schema->has_inverted_index(*column)) { @@ -141,9 +149,12 @@ Status SegcompactionWorker::_delete_original_segments(uint32_t begin, uint32_t e auto idx_path = InvertedIndexDescriptor::inverted_index_file_path( ctx.rowset_dir, ctx.rowset_id, i, index_id, index_info->get_index_suffix()); VLOG_DEBUG << "segcompaction index. delete file " << idx_path; - RETURN_NOT_OK_STATUS_WITH_WARN( - fs->delete_file(idx_path), - strings::Substitute("Failed to delete file=$0", idx_path)); + if (schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::V1) { + RETURN_NOT_OK_STATUS_WITH_WARN( + fs->delete_file(idx_path), + strings::Substitute("Failed to delete file=$0", idx_path)); + } // Erase the origin index file cache RETURN_IF_ERROR(InvertedIndexSearcherCache::instance()->erase(idx_path)); } diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 65896044dd..545a1ac8a3 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -41,6 +41,7 @@ #include "olap/rowset/segment_v2/bloom_filter.h" #include "olap/rowset/segment_v2/bloom_filter_index_reader.h" #include "olap/rowset/segment_v2/encoding_info.h" // for EncodingInfo +#include "olap/rowset/segment_v2/inverted_index_file_reader.h" #include "olap/rowset/segment_v2/inverted_index_reader.h" #include "olap/rowset/segment_v2/page_decoder.h" #include "olap/rowset/segment_v2/page_handle.h" // for PageHandle @@ -257,10 +258,10 @@ Status ColumnReader::new_bitmap_index_iterator(BitmapIndexIterator** iterator) { return Status::OK(); } -Status ColumnReader::new_inverted_index_iterator(const TabletIndex* index_meta, - const StorageReadOptions& read_options, - std::unique_ptr* iterator) { - RETURN_IF_ERROR(_ensure_inverted_index_loaded(index_meta)); +Status ColumnReader::new_inverted_index_iterator( + std::shared_ptr index_file_reader, const TabletIndex* index_meta, + const StorageReadOptions& read_options, std::unique_ptr* iterator) { + RETURN_IF_ERROR(_ensure_inverted_index_loaded(index_file_reader, index_meta)); if (_inverted_index) { RETURN_IF_ERROR(_inverted_index->new_iterator(read_options.stats, read_options.runtime_state, iterator)); @@ -533,7 +534,8 @@ Status ColumnReader::_load_bitmap_index(bool use_page_cache, bool kept_in_memory return Status::OK(); } -Status ColumnReader::_load_inverted_index_index(const TabletIndex* index_meta) { +Status ColumnReader::_load_inverted_index_index( + std::shared_ptr index_file_reader, const TabletIndex* index_meta) { std::lock_guard wlock(_load_index_lock); if (_inverted_index && index_meta && @@ -553,16 +555,15 @@ Status ColumnReader::_load_inverted_index_index(const TabletIndex* index_meta) { if (is_string_type(type)) { if (parser_type != InvertedIndexParserType::PARSER_NONE) { try { - _inverted_index = FullTextIndexReader::create_shared( - _file_reader->fs(), _file_reader->path().native(), index_meta); + _inverted_index = FullTextIndexReader::create_shared(index_meta, index_file_reader); } catch (const CLuceneError& e) { return Status::Error( "create FullTextIndexReader error: {}", e.what()); } } else { try { - _inverted_index = StringTypeInvertedIndexReader::create_shared( - _file_reader->fs(), _file_reader->path().native(), index_meta); + _inverted_index = + StringTypeInvertedIndexReader::create_shared(index_meta, index_file_reader); } catch (const CLuceneError& e) { return Status::Error( "create StringTypeInvertedIndexReader error: {}", e.what()); @@ -570,8 +571,7 @@ Status ColumnReader::_load_inverted_index_index(const TabletIndex* index_meta) { } } else if (is_numeric_type(type)) { try { - _inverted_index = BkdIndexReader::create_shared( - _file_reader->fs(), _file_reader->path().native(), index_meta); + _inverted_index = BkdIndexReader::create_shared(index_meta, index_file_reader); } catch (const CLuceneError& e) { return Status::Error( "create BkdIndexReader error: {}", e.what()); @@ -579,6 +579,10 @@ Status ColumnReader::_load_inverted_index_index(const TabletIndex* index_meta) { } else { _inverted_index.reset(); } + // TODO: move has null to inverted_index_reader's query function + //bool has_null = true; + //RETURN_IF_ERROR(index_file_reader->has_null(index_meta, &has_null)); + //_inverted_index->set_has_null(has_null); return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index edfe4fbfce..9ef9d53d66 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -72,6 +72,7 @@ class BitmapIndexIterator; class BitmapIndexReader; class InvertedIndexIterator; class InvertedIndexReader; +class InvertedIndexFileReader; class PageDecoder; class RowRanges; class ZoneMapIndexReader; @@ -122,7 +123,8 @@ public: // Client should delete returned iterator Status new_bitmap_index_iterator(BitmapIndexIterator** iterator); - Status new_inverted_index_iterator(const TabletIndex* index_meta, + Status new_inverted_index_iterator(std::shared_ptr index_file_reader, + const TabletIndex* index_meta, const StorageReadOptions& read_options, std::unique_ptr* iterator); @@ -191,16 +193,19 @@ private: // Read column inverted indexes into memory // May be called multiple times, subsequent calls will no op. - Status _ensure_inverted_index_loaded(const TabletIndex* index_meta) { + Status _ensure_inverted_index_loaded(std::shared_ptr index_file_reader, + const TabletIndex* index_meta) { // load inverted index only if not loaded or index_id is changed - RETURN_IF_ERROR(_load_inverted_index_index(index_meta)); + RETURN_IF_ERROR(_load_inverted_index_index(index_file_reader, index_meta)); return Status::OK(); } [[nodiscard]] Status _load_zone_map_index(bool use_page_cache, bool kept_in_memory); [[nodiscard]] Status _load_ordinal_index(bool use_page_cache, bool kept_in_memory); [[nodiscard]] Status _load_bitmap_index(bool use_page_cache, bool kept_in_memory); - [[nodiscard]] Status _load_inverted_index_index(const TabletIndex* index_meta); + [[nodiscard]] Status _load_inverted_index_index( + std::shared_ptr index_file_reader, + const TabletIndex* index_meta); [[nodiscard]] Status _load_bloom_filter_index(bool use_page_cache, bool kept_in_memory); bool _zone_map_match_condition(const ZoneMapPB& zone_map, WrapperField* min_value_container, diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp index 99299e3ce7..6bc4d4d336 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.cpp +++ b/be/src/olap/rowset/segment_v2/column_writer.cpp @@ -113,7 +113,6 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn* column_options.need_zone_map = false; column_options.need_bloom_filter = sub_column.is_bf_column(); column_options.need_bitmap_index = sub_column.has_bitmap_index(); - column_options.inverted_index = nullptr; if (sub_column.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) { if (column_options.need_bloom_filter) { return Status::NotSupported("Do not support bloom filter for struct type"); @@ -182,7 +181,6 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn* item_options.need_zone_map = false; item_options.need_bloom_filter = item_column.is_bf_column(); item_options.need_bitmap_index = item_column.has_bitmap_index(); - item_options.inverted_index = nullptr; if (item_column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { if (item_options.need_bloom_filter) { return Status::NotSupported("Do not support bloom filter for array type"); @@ -271,7 +269,6 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn* item_options.need_zone_map = false; item_options.need_bloom_filter = item_column.is_bf_column(); item_options.need_bitmap_index = item_column.has_bitmap_index(); - item_options.inverted_index = nullptr; if (item_column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { if (item_options.need_bloom_filter) { return Status::NotSupported("Do not support bloom filter for map type"); @@ -483,11 +480,10 @@ Status ScalarColumnWriter::init() { BitmapIndexWriter::create(get_field()->type_info(), &_bitmap_index_builder)); } - if (_opts.inverted_index) { - RETURN_IF_ERROR(InvertedIndexColumnWriter::create( - get_field(), &_inverted_index_builder, _file_writer->path().filename().native(), - _file_writer->path().parent_path().native(), _opts.inverted_index, - _file_writer->fs())); + if (_opts.need_inverted_index) { + RETURN_IF_ERROR(InvertedIndexColumnWriter::create(get_field(), &_inverted_index_builder, + _opts.inverted_index_file_writer, + _opts.inverted_index)); } if (_opts.need_bloom_filter) { if (_opts.is_ngram_bf_index) { @@ -511,7 +507,7 @@ Status ScalarColumnWriter::append_nulls(size_t num_rows) { if (_opts.need_bitmap_index) { _bitmap_index_builder->add_nulls(num_rows); } - if (_opts.inverted_index) { + if (_opts.need_inverted_index) { RETURN_IF_ERROR(_inverted_index_builder->add_nulls(num_rows)); } if (_opts.need_bloom_filter) { @@ -546,7 +542,7 @@ Status ScalarColumnWriter::append_data_in_current_page(const uint8_t* data, size if (_opts.need_bitmap_index) { _bitmap_index_builder->add_values(data, *num_written); } - if (_opts.inverted_index) { + if (_opts.need_inverted_index) { RETURN_IF_ERROR( _inverted_index_builder->add_values(get_field()->name(), data, *num_written)); } @@ -639,14 +635,14 @@ Status ScalarColumnWriter::write_bitmap_index() { } Status ScalarColumnWriter::write_inverted_index() { - if (_opts.inverted_index) { + if (_opts.need_inverted_index) { return _inverted_index_builder->finish(); } return Status::OK(); } size_t ScalarColumnWriter::get_inverted_index_size() { - if (_opts.inverted_index) { + if (_opts.need_inverted_index) { auto size = _inverted_index_builder->file_size(); return size == -1 ? 0 : size; } @@ -800,7 +796,7 @@ Status StructColumnWriter::init() { } Status StructColumnWriter::write_inverted_index() { - if (_opts.inverted_index) { + if (_opts.need_inverted_index) { for (auto& column_writer : _sub_column_writers) { RETURN_IF_ERROR(column_writer->write_inverted_index()); } @@ -810,7 +806,7 @@ Status StructColumnWriter::write_inverted_index() { size_t StructColumnWriter::get_inverted_index_size() { size_t total_size = 0; - if (_opts.inverted_index) { + if (_opts.need_inverted_index) { for (auto& column_writer : _sub_column_writers) { auto size = column_writer->get_inverted_index_size(); total_size += (size == -1 ? 0 : size); @@ -915,28 +911,26 @@ Status ArrayColumnWriter::init() { RETURN_IF_ERROR(_null_writer->init()); } RETURN_IF_ERROR(_item_writer->init()); - if (_opts.inverted_index) { + if (_opts.need_inverted_index) { auto writer = dynamic_cast(_item_writer.get()); if (writer != nullptr) { - RETURN_IF_ERROR(InvertedIndexColumnWriter::create( - get_field(), &_inverted_index_builder, - writer->_file_writer->path().filename().native(), - writer->_file_writer->path().parent_path().native(), _opts.inverted_index, - writer->_file_writer->fs())); + RETURN_IF_ERROR(InvertedIndexColumnWriter::create(get_field(), &_inverted_index_builder, + _opts.inverted_index_file_writer, + _opts.inverted_index)); } } return Status::OK(); } Status ArrayColumnWriter::write_inverted_index() { - if (_opts.inverted_index) { + if (_opts.need_inverted_index) { return _inverted_index_builder->finish(); } return Status::OK(); } size_t ArrayColumnWriter::get_inverted_index_size() { - if (_opts.inverted_index) { + if (_opts.need_inverted_index) { auto size = _inverted_index_builder->file_size(); return size == -1 ? 0 : size; } @@ -958,7 +952,7 @@ Status ArrayColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) { RETURN_IF_ERROR(_item_writer->append(reinterpret_cast(nested_null_map), reinterpret_cast(data), element_cnt)); } - if (_opts.inverted_index) { + if (_opts.need_inverted_index) { auto writer = dynamic_cast(_item_writer.get()); // now only support nested type is scala if (writer != nullptr) { @@ -1177,14 +1171,14 @@ Status MapColumnWriter::finish_current_page() { } Status MapColumnWriter::write_inverted_index() { - if (_opts.inverted_index) { + if (_opts.need_inverted_index) { return _inverted_index_builder->finish(); } return Status::OK(); } size_t MapColumnWriter::get_inverted_index_size() { - if (_opts.inverted_index) { + if (_opts.need_inverted_index) { auto size = _inverted_index_builder->file_size(); return size == -1 ? 0 : size; } diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h index ae22245b14..1f60b006e5 100644 --- a/be/src/olap/rowset/segment_v2/column_writer.h +++ b/be/src/olap/rowset/segment_v2/column_writer.h @@ -60,10 +60,12 @@ struct ColumnWriterOptions { bool need_bitmap_index = false; bool need_bloom_filter = false; bool is_ngram_bf_index = false; + bool need_inverted_index = false; uint8_t gram_size; uint16_t gram_bf_size; std::vector indexes; const TabletIndex* inverted_index = nullptr; + InvertedIndexFileWriter* inverted_index_file_writer; std::string to_string() const { std::stringstream ss; ss << std::boolalpha << "meta=" << meta->DebugString() diff --git a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp index fe456d53ea..60f32883d8 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp @@ -17,28 +17,17 @@ #include "olap/rowset/segment_v2/inverted_index_cache.h" -#include -#include -#include // IWYU pragma: no_include #include -#include // IWYU pragma: keep #include // IWYU pragma: no_include -#include // IWYU pragma: keep #include #include -#include "common/logging.h" -#include "olap/olap_common.h" -#include "olap/rowset/segment_v2/inverted_index_compound_directory.h" -#include "olap/rowset/segment_v2/inverted_index_compound_reader.h" -#include "olap/rowset/segment_v2/inverted_index_desc.h" #include "runtime/exec_env.h" #include "runtime/thread_context.h" #include "util/defer_op.h" -#include "util/runtime_profile.h" namespace doris::segment_v2 { diff --git a/be/src/olap/rowset/segment_v2/inverted_index_cache.h b/be/src/olap/rowset/segment_v2/inverted_index_cache.h index 3292b10e25..1386ee7fab 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_cache.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_cache.h @@ -17,16 +17,6 @@ #pragma once -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wshadow-field" -#endif -#include // IWYU pragma: keep -#ifdef __clang__ -#pragma clang diagnostic pop -#endif -#include -#include #include #include #include @@ -34,18 +24,14 @@ #include #include -#include #include #include -#include -#include #include "common/config.h" #include "common/status.h" #include "io/fs/file_system.h" #include "io/fs/path.h" #include "olap/lru_cache.h" -#include "olap/rowset/segment_v2/inverted_index_query_type.h" #include "olap/rowset/segment_v2/inverted_index_searcher.h" #include "runtime/exec_env.h" #include "runtime/memory/lru_cache_policy.h" @@ -53,23 +39,9 @@ #include "util/slice.h" #include "util/time.h" -namespace lucene { -namespace search { -class IndexSearcher; -} // namespace search - -namespace util::bkd { -class bkd_reader; -} - -} // namespace lucene - namespace doris { -struct OlapReaderStatistics; - namespace segment_v2 { class InvertedIndexCacheHandle; -class DorisCompoundReader; class InvertedIndexSearcherCache { public: diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp index ea019ef0ad..1f988a7584 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp @@ -17,17 +17,15 @@ #include "inverted_index_compaction.h" -#include - -#include "inverted_index_compound_directory.h" -#include "inverted_index_compound_reader.h" +#include "inverted_index_file_writer.h" +#include "inverted_index_fs_directory.h" +#include "olap/tablet_schema.h" #include "util/debug_points.h" namespace doris::segment_v2 { -Status compact_column(int32_t index_id, int src_segment_num, int dest_segment_num, - std::vector src_index_files, - std::vector dest_index_files, const io::FileSystemSPtr& fs, - std::string index_writer_path, std::string tablet_path, +Status compact_column(int64_t index_id, std::vector& src_index_dirs, + std::vector& dest_index_dirs, + const io::FileSystemSPtr& fs, std::string tmp_path, std::vector>> trans_vec, std::vector dest_segment_num_rows) { DBUG_EXECUTE_IF("index_compaction_compact_column_throw_error", { @@ -41,32 +39,12 @@ Status compact_column(int32_t index_id, int src_segment_num, int dest_segment_nu "debug point: index compaction error"); } }) - lucene::store::Directory* dir = - DorisCompoundDirectoryFactory::getDirectory(fs, index_writer_path.c_str()); + + lucene::store::Directory* dir = DorisFSDirectoryFactory::getDirectory(fs, tmp_path.c_str()); lucene::analysis::SimpleAnalyzer analyzer; auto* index_writer = _CLNEW lucene::index::IndexWriter(dir, &analyzer, true /* create */, true /* closeDirOnShutdown */); - // get compound directory src_index_dirs - std::vector src_index_dirs(src_segment_num); - for (int i = 0; i < src_segment_num; ++i) { - // format: rowsetId_segmentId_indexId.idx - std::string src_idx_full_name = - src_index_files[i] + "_" + std::to_string(index_id) + ".idx"; - auto* reader = new DorisCompoundReader( - DorisCompoundDirectoryFactory::getDirectory(fs, tablet_path.c_str()), - src_idx_full_name.c_str()); - src_index_dirs[i] = reader; - } - - // get dest idx file paths - std::vector dest_index_dirs(dest_segment_num); - for (int i = 0; i < dest_segment_num; ++i) { - // format: rowsetId_segmentId_columnId - auto path = tablet_path + "/" + dest_index_files[i] + "_" + std::to_string(index_id); - dest_index_dirs[i] = DorisCompoundDirectoryFactory::getDirectory(fs, path.c_str(), true); - } - DCHECK_EQ(src_index_dirs.size(), trans_vec.size()); index_writer->indexCompaction(src_index_dirs, dest_index_dirs, trans_vec, dest_segment_num_rows); @@ -80,19 +58,19 @@ Status compact_column(int32_t index_id, int src_segment_num, int dest_segment_nu for (auto* d : src_index_dirs) { if (d != nullptr) { d->close(); - _CLDELETE(d); + //_CLDELETE(d); } } for (auto* d : dest_index_dirs) { if (d != nullptr) { // NOTE: DO NOT close dest dir here, because it will be closed when dest index writer finalize. //d->close(); - _CLDELETE(d); + //_CLDELETE(d); } } - // delete temporary index_writer_path - RETURN_IF_ERROR(fs->delete_directory(index_writer_path.c_str())); + // delete temporary segment_path + static_cast(fs->delete_directory(tmp_path.c_str())); return Status::OK(); } } // namespace doris::segment_v2 diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h b/be/src/olap/rowset/segment_v2/inverted_index_compaction.h index f615192b19..3dadd7f476 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_compaction.h @@ -16,6 +16,8 @@ // under the License. #pragma once +#include + #include #include #include @@ -23,12 +25,14 @@ #include "io/fs/file_system.h" namespace doris { - +class TabletIndex; namespace segment_v2 { -Status compact_column(int32_t index_id, int src_segment_num, int dest_segment_num, - std::vector src_index_files, - std::vector dest_index_files, const io::FileSystemSPtr& fs, - std::string index_writer_path, std::string tablet_path, +class InvertedIndexFileWriter; +class InvertedIndexFileReader; + +Status compact_column(int64_t index_id, std::vector& src_index_dirs, + std::vector& dest_index_dirs, + const io::FileSystemSPtr& fs, std::string tmp_path, std::vector>> trans_vec, std::vector dest_segment_num_rows); } // namespace segment_v2 diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp index d8b204f98a..540053fc9b 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp @@ -26,12 +26,12 @@ #include #include -#include #include #include #include "CLucene/SharedHeader.h" -#include "olap/rowset/segment_v2/inverted_index_compound_directory.h" +#include "olap/rowset/segment_v2/inverted_index_fs_directory.h" +#include "olap/tablet_schema.h" namespace doris { namespace io { @@ -50,19 +50,6 @@ using FileWriterPtr = std::unique_ptr; namespace doris { namespace segment_v2 { -class DorisCompoundReader::ReaderFileEntry : LUCENE_BASE { -public: - std::string file_name {}; - int64_t offset; - int64_t length; - ReaderFileEntry() { - //file_name = nullptr; - offset = 0; - length = 0; - } - ~ReaderFileEntry() override = default; -}; - /** Implementation of an IndexInput that reads from a portion of the * compound file. */ @@ -98,7 +85,7 @@ CSIndexInput::CSIndexInput(CL_NS(store)::IndexInput* base, const int64_t fileOff } void CSIndexInput::readInternal(uint8_t* b, const int32_t len) { - std::lock_guard wlock(((DorisCompoundDirectory::FSIndexInput*)base)->_this_lock); + std::lock_guard wlock(((DorisFSDirectory::FSIndexInput*)base)->_this_lock); int64_t start = getFilePointer(); if (start + len > _length) { @@ -210,7 +197,9 @@ void DorisCompoundReader::copyFile(const char* file, int64_t file_length, uint8_ } DorisCompoundReader::~DorisCompoundReader() { - _CLDELETE(entries) + if (_own_index_input) { + _CLDELETE(entries) + } } const char* DorisCompoundReader::getClassName() { @@ -236,7 +225,7 @@ lucene::store::Directory* DorisCompoundReader::getDirectory() { } std::string DorisCompoundReader::getPath() const { - return ((DorisCompoundDirectory*)dir)->getCfsDirName(); + return ((DorisFSDirectory*)dir)->getCfsDirName(); } int64_t DorisCompoundReader::fileModified(const char* name) const { @@ -282,7 +271,7 @@ bool DorisCompoundReader::openInput(const char* name, lucene::store::IndexInput* } // If file is in RAM, just return. - if (ram_dir->fileExists(name)) { + if (ram_dir && ram_dir->fileExists(name)) { return ram_dir->openInput(name, ret, error, bufferSize); } @@ -296,15 +285,19 @@ bool DorisCompoundReader::openInput(const char* name, lucene::store::IndexInput* void DorisCompoundReader::close() { std::lock_guard wlock(_this_lock); - if (stream != nullptr) { + if (_own_index_input && stream != nullptr) { entries->clear(); stream->close(); _CLDELETE(stream) } - ram_dir->close(); - dir->close(); - _CLDECDELETE(dir) - _CLDELETE(ram_dir) + if (ram_dir) { + ram_dir->close(); + _CLDELETE(ram_dir) + } + if (dir) { + dir->close(); + _CLDECDELETE(dir) + } } bool DorisCompoundReader::doDeleteFile(const char* /*name*/) { diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h index 762987f91c..4786b8f35e 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h @@ -30,8 +30,12 @@ #include #include #include +#include #include +#include "io/fs/file_system.h" +#include "olap/rowset/segment_v2/inverted_index_desc.h" + class CLuceneError; namespace lucene { @@ -41,15 +45,28 @@ class RAMDirectory; } // namespace lucene namespace doris { - +class TabletIndex; namespace segment_v2 { +class ReaderFileEntry : LUCENE_BASE { +public: + std::string file_name {}; + int64_t offset; + int64_t length; + ReaderFileEntry() { + //file_name = nullptr; + offset = 0; + length = 0; + } + ~ReaderFileEntry() override = default; +}; + +using EntriesType = + lucene::util::CLHashMap>; class CLUCENE_EXPORT DorisCompoundReader : public lucene::store::Directory { private: - class ReaderFileEntry; - - friend class DorisCompoundReader::ReaderFileEntry; - int32_t readBufferSize; // base info lucene::store::Directory* dir = nullptr; @@ -57,21 +74,33 @@ private: std::string directory; std::string file_name; CL_NS(store)::IndexInput* stream = nullptr; - - using EntriesType = - lucene::util::CLHashMap>; - EntriesType* entries = nullptr; - std::mutex _this_lock; + bool _own_index_input = true; protected: /** Removes an existing file in the directory-> */ bool doDeleteFile(const char* name) override; public: + explicit DorisCompoundReader( + CL_NS(store)::IndexInput* stream, EntriesType* entries_clone, + bool own_index_input = false, + int32_t _readBufferSize = CL_NS(store)::BufferedIndexInput::BUFFER_SIZE) + : readBufferSize(_readBufferSize), + stream(stream), + entries(_CLNEW EntriesType(true, true)), + _own_index_input(own_index_input) { + for (auto& e : *entries_clone) { + auto* origin_entry = e.second; + auto* entry = _CLNEW ReaderFileEntry(); + char* aid = strdup(e.first); + entry->file_name = origin_entry->file_name; + entry->offset = origin_entry->offset; + entry->length = origin_entry->length; + entries->put(aid, entry); + } + }; DorisCompoundReader(lucene::store::Directory* dir, const char* name, int32_t _readBufferSize = CL_NS(store)::BufferedIndexInput::BUFFER_SIZE, bool open_idx_file_cache = false); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp index bf05b85751..1544d18ed7 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp @@ -23,9 +23,9 @@ #include "olap/olap_common.h" namespace doris::segment_v2 { -const std::string segment_suffix = ".dat"; -const std::string index_suffix = ".idx"; -const std::string index_name_separator = "_"; +const std::string InvertedIndexDescriptor::segment_suffix = ".dat"; +const std::string InvertedIndexDescriptor::index_suffix = ".idx"; +const std::string InvertedIndexDescriptor::index_name_separator = "_"; std::string InvertedIndexDescriptor::get_temporary_index_path( const std::string& segment_path, uint32_t uuid, const std::string& index_suffix_path) { @@ -51,6 +51,10 @@ std::string InvertedIndexDescriptor::inverted_index_file_path( suffix); } +std::string InvertedIndexDescriptor::get_index_file_name(const std::string& segment_file_name) { + return StripSuffixString(segment_file_name, segment_suffix) + index_suffix; +} + std::string InvertedIndexDescriptor::local_inverted_index_path_segcompacted( const string& tablet_path, const RowsetId& rowset_id, int64_t begin, int64_t end, int64_t index_id, const std::string& index_suffix_path) { diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.h b/be/src/olap/rowset/segment_v2/inverted_index_desc.h index 25c9913e21..5d86e7ddd3 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_desc.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.h @@ -28,10 +28,14 @@ namespace segment_v2 { class InvertedIndexDescriptor { public: + static const std::string segment_suffix; + static const std::string index_suffix; + static const std::string index_name_separator; static std::string get_temporary_index_path(const std::string& segment_path, uint32_t uuid, const std::string& index_suffix_path); static std::string get_index_file_name(const std::string& path, uint32_t uuid, const std::string& index_suffix_path); + static std::string get_index_file_name(const std::string& path); static const std::string get_temporary_null_bitmap_file_name() { return "null_bitmap"; } static const std::string get_temporary_bkd_index_data_file_name() { return "bkd"; } static const std::string get_temporary_bkd_index_meta_file_name() { return "bkd_meta"; } diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp new file mode 100644 index 0000000000..03db293f61 --- /dev/null +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/rowset/segment_v2/inverted_index_file_reader.h" + +#include +#include + +#include "olap/rowset/segment_v2/inverted_index_compound_reader.h" +#include "olap/rowset/segment_v2/inverted_index_fs_directory.h" +#include "olap/tablet_schema.h" + +namespace doris::segment_v2 { + +Status InvertedIndexFileReader::init(int32_t read_buffer_size, bool open_idx_file_cache) { + _read_buffer_size = read_buffer_size; + _open_idx_file_cache = open_idx_file_cache; + if (_storage_format == InvertedIndexStorageFormatPB::V2) { + return _init_from_v2(read_buffer_size); + } else { + return Status::OK(); + } +} + +Status InvertedIndexFileReader::_init_from_v2(int32_t read_buffer_size) { + std::unique_lock lock(_mutex); // Lock for writing + auto index_file_full_path = _index_file_dir / _index_file_name; + try { + bool exists = false; + RETURN_IF_ERROR(_fs->exists(index_file_full_path, &exists)); + if (!exists) { + return Status::Error( + "inverted index file {} is not found", index_file_full_path.native()); + } + int64_t file_size = 0; + RETURN_IF_ERROR(_fs->file_size(index_file_full_path, &file_size)); + if (file_size == 0) { + LOG(WARNING) << "inverted index file " << index_file_full_path << " is empty."; + return Status::Error( + "inverted index file {} is empty", index_file_full_path.native()); + } + + CLuceneError err; + CL_NS(store)::IndexInput* index_input = nullptr; + auto ok = DorisFSDirectory::FSIndexInput::open(_fs, index_file_full_path.c_str(), + index_input, err, read_buffer_size); + if (!ok) { + return Status::Error( + "CLuceneError occur when open idx file {}, error msg: {}", + index_file_full_path.native(), err.what()); + } + index_input->setIdxFileCache(_open_idx_file_cache); + _stream = std::unique_ptr(index_input); + int32_t version = _stream->readInt(); // Read version number + if (version == InvertedIndexStorageFormatPB::V2) { + DCHECK(version == _storage_format); + int32_t numIndices = _stream->readInt(); // Read number of indices + ReaderFileEntry* entry = nullptr; + + for (int32_t i = 0; i < numIndices; ++i) { + int64_t indexId = _stream->readInt(); // Read index ID + int32_t suffix_length = _stream->readInt(); // Read suffix length + std::vector suffix_data(suffix_length); + _stream->readBytes(suffix_data.data(), suffix_length); + std::string suffix_str(suffix_data.begin(), suffix_data.end()); + + int32_t numFiles = _stream->readInt(); // Read number of files in the index + + // true, true means it will deconstruct key and value + auto fileEntries = std::make_unique(true, true); + + for (int32_t j = 0; j < numFiles; ++j) { + entry = _CLNEW ReaderFileEntry(); + + int32_t file_name_length = _stream->readInt(); + // aid will destruct in EntriesType map. + char* aid = (char*)malloc(file_name_length + 1); + _stream->readBytes(reinterpret_cast(aid), file_name_length); + aid[file_name_length] = '\0'; + //stream->readString(tid, CL_MAX_PATH); + entry->file_name = std::string(aid, file_name_length); + entry->offset = _stream->readLong(); + entry->length = _stream->readLong(); + + fileEntries->put(aid, entry); + } + + _indices_entries.emplace(std::make_pair(indexId, std::move(suffix_str)), + std::move(fileEntries)); + } + } else { + return Status::Error( + "unknown inverted index format {}", version); + } + } catch (CLuceneError& err) { + if (_stream != nullptr) { + try { + _stream->close(); + } catch (CLuceneError& err) { + return Status::Error( + "CLuceneError occur when close idx file {}, error msg: {}", + index_file_full_path.native(), err.what()); + } + } + return Status::Error( + "CLuceneError occur when init idx file {}, error msg: {}", + index_file_full_path.native(), err.what()); + } + return Status::OK(); +} + +Result InvertedIndexFileReader::get_all_directories() { + InvertedIndexDirectoryMap res; + std::shared_lock lock(_mutex); // Lock for reading + for (auto& index : _indices_entries) { + auto index_id = index.first.first; + auto index_suffix = index.first.second; + LOG(INFO) << "index_id:" << index_id << " index_suffix:" << index_suffix; + auto ret = _open(index_id, index_suffix); + if (!ret.has_value()) { + return ResultError(ret.error()); + } + res.emplace(std::make_pair(index_id, index_suffix), std::move(ret.value())); + } + return res; +} + +Result> InvertedIndexFileReader::_open( + int64_t index_id, std::string& index_suffix) const { + std::unique_ptr compound_reader; + + if (_storage_format == InvertedIndexStorageFormatPB::V1) { + DorisFSDirectory* dir = nullptr; + auto file_name = InvertedIndexDescriptor::get_index_file_name(_segment_file_name, index_id, + index_suffix); + try { + dir = DorisFSDirectoryFactory::getDirectory(_fs, _index_file_dir.c_str()); + + compound_reader = std::make_unique( + dir, file_name.c_str(), _read_buffer_size, _open_idx_file_cache); + } catch (CLuceneError& err) { + if (dir != nullptr) { + dir->close(); + _CLDELETE(dir) + } + return ResultError(Status::Error( + "CLuceneError occur when open idx file {}, error msg: {}", + (_index_file_dir / file_name).native(), err.what())); + } + } else { + std::shared_lock lock(_mutex); // Lock for reading + if (_stream == nullptr) { + return ResultError(Status::Error( + "CLuceneError occur when open idx file {}, stream is nullptr", + (_index_file_dir / _index_file_name).native())); + } + + // Check if the specified index exists + auto index_it = _indices_entries.find(std::make_pair(index_id, index_suffix)); + if (index_it == _indices_entries.end()) { + std::ostringstream errMsg; + errMsg << "No index with id " << index_id << " found"; + return ResultError(Status::Error( + "CLuceneError occur when open idx file {}, error msg: {}", + (_index_file_dir / _index_file_name).native(), errMsg.str())); + } + // Need to clone resource here, because index searcher cache need it. + bool own_index_input = true; + compound_reader = std::make_unique( + _stream->clone(), index_it->second.get(), own_index_input, _read_buffer_size); + } + return compound_reader; +} +Result> InvertedIndexFileReader::open( + const TabletIndex* index_meta) const { + auto index_id = index_meta->index_id(); + auto index_suffix = index_meta->get_index_suffix(); + return _open(index_id, index_suffix); +} + +std::string InvertedIndexFileReader::get_index_file_path(const TabletIndex* index_meta) const { + return InvertedIndexDescriptor::get_index_file_name(_index_file_dir / _segment_file_name, + index_meta->index_id(), + index_meta->get_index_suffix()); +} + +Status InvertedIndexFileReader::index_file_exist(const TabletIndex* index_meta, bool* res) const { + if (_storage_format == InvertedIndexStorageFormatPB::V1) { + auto index_file_path = _index_file_dir / InvertedIndexDescriptor::get_index_file_name( + _segment_file_name, index_meta->index_id(), + index_meta->get_index_suffix()); + return _fs->exists(index_file_path, res); + } else { + std::shared_lock lock(_mutex); // Lock for reading + if (_stream == nullptr) { + *res = false; + return Status::Error( + "idx file {} is not opened", (_index_file_dir / _index_file_name).native()); + } + // Check if the specified index exists + auto index_it = _indices_entries.find( + std::make_pair(index_meta->index_id(), index_meta->get_index_suffix())); + if (index_it == _indices_entries.end()) { + *res = false; + } else { + *res = true; + } + } + return Status::OK(); +} + +Status InvertedIndexFileReader::has_null(const TabletIndex* index_meta, bool* res) const { + if (_storage_format == InvertedIndexStorageFormatPB::V1) { + *res = true; + return Status::OK(); + } + std::shared_lock lock(_mutex); // Lock for reading + if (_stream == nullptr) { + return Status::Error( + "idx file {} is not opened", (_index_file_dir / _index_file_name).native()); + } + // Check if the specified index exists + auto index_it = _indices_entries.find( + std::make_pair(index_meta->index_id(), index_meta->get_index_suffix())); + if (index_it == _indices_entries.end()) { + *res = false; + } else { + auto null_bitmap_file_name = InvertedIndexDescriptor::get_temporary_null_bitmap_file_name(); + auto* entries = index_it->second.get(); + ReaderFileEntry* e = entries->get((char*)(null_bitmap_file_name.c_str())); + if (e == nullptr) { + *res = false; + return Status::OK(); + } + // roaring bitmap cookie header size is 5 + if (e->length <= 5) { + *res = false; + } else { + *res = true; + } + } + return Status::OK(); +} + +void InvertedIndexFileReader::debug_file_entries() { + std::shared_lock lock(_mutex); // Lock for reading + for (auto& index : _indices_entries) { + LOG(INFO) << "index_id:" << index.first.first; + auto* index_entries = index.second.get(); + for (auto& entry : (*index_entries)) { + ReaderFileEntry* file_entry = entry.second; + LOG(INFO) << "file entry name:" << file_entry->file_name + << " length:" << file_entry->length << " offset:" << file_entry->offset; + } + } +} + +} // namespace doris::segment_v2 diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h new file mode 100644 index 0000000000..c87ea9b237 --- /dev/null +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include // IWYU pragma: keep +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/config.h" +#include "io/fs/file_system.h" +#include "olap/rowset/segment_v2/inverted_index_desc.h" +#include "olap/rowset/segment_v2/inverted_index_file_writer.h" + +namespace doris { +class TabletIndex; +namespace segment_v2 { +class ReaderFileEntry; +class DorisCompoundReader; + +class InvertedIndexFileReader { +public: + using EntriesType = + lucene::util::CLHashMap>; + // Map to hold the file entries for each index ID. + using IndicesEntriesMap = + std::map, std::unique_ptr>; + + InvertedIndexFileReader(const io::FileSystemSPtr& fs, io::Path segment_file_dir, + std::string segment_file_name, + InvertedIndexStorageFormatPB storage_format) + : _fs(fs), + _index_file_dir(std::move(segment_file_dir)), + _segment_file_name(std::move(segment_file_name)), + _storage_format(storage_format) { + if (_storage_format != InvertedIndexStorageFormatPB::V1) { + _index_file_name = InvertedIndexDescriptor::get_index_file_name(_segment_file_name); + } + } + + Status init(int32_t read_buffer_size = config::inverted_index_read_buffer_size, + bool open_idx_file_cache = false); + Result> open(const TabletIndex* index_meta) const; + void debug_file_entries(); + std::string get_index_file_path(const TabletIndex* index_meta) const; + Status index_file_exist(const TabletIndex* index_meta, bool* res) const; + Status has_null(const TabletIndex* index_meta, bool* res) const; + Result get_all_directories(); + +private: + Status _init_from_v2(int32_t read_buffer_size); + Result> _open(int64_t index_id, + std::string& index_suffix) const; + + IndicesEntriesMap _indices_entries; + std::unique_ptr _stream; + const io::FileSystemSPtr _fs; + io::Path _index_file_dir; + // index file name by storage format V2 + std::string _index_file_name; + std::string _segment_file_name; + int32_t _read_buffer_size = -1; + bool _open_idx_file_cache = false; + InvertedIndexStorageFormatPB _storage_format; + mutable std::shared_mutex _mutex; // Use mutable for const read operations +}; + +} // namespace segment_v2 +} // namespace doris \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp new file mode 100644 index 0000000000..0ae8a1d7af --- /dev/null +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp @@ -0,0 +1,414 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/rowset/segment_v2/inverted_index_file_writer.h" + +#include "common/status.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" +#include "olap/rowset/segment_v2/inverted_index_desc.h" +#include "olap/rowset/segment_v2/inverted_index_fs_directory.h" +#include "olap/tablet_schema.h" +#include "runtime/exec_env.h" + +namespace doris::segment_v2 { + +std::string InvertedIndexFileWriter::get_index_file_path(const TabletIndex* index_meta) const { + return InvertedIndexDescriptor::get_index_file_name(_index_file_dir / _segment_file_name, + index_meta->index_id(), + index_meta->get_index_suffix()); +} + +Status InvertedIndexFileWriter::initialize(InvertedIndexDirectoryMap& indices_dirs) { + _indices_dirs = std::move(indices_dirs); + return Status::OK(); +} + +Result InvertedIndexFileWriter::open(const TabletIndex* index_meta) { + auto index_id = index_meta->index_id(); + auto index_suffix = index_meta->get_index_suffix(); + auto tmp_file_dir = ExecEnv::GetInstance()->get_tmp_file_dirs()->get_tmp_file_dir(); + _lfs = io::global_local_filesystem(); + auto lfs_index_path = InvertedIndexDescriptor::get_temporary_index_path( + tmp_file_dir / _segment_file_name, index_meta->index_id(), + index_meta->get_index_suffix()); + auto index_path = InvertedIndexDescriptor::get_temporary_index_path( + (_index_file_dir / _segment_file_name).native(), index_id, index_suffix); + + bool exists = false; + auto st = _fs->exists(lfs_index_path.c_str(), &exists); + if (!st.ok()) { + LOG(ERROR) << "index_path:" << lfs_index_path << " exists error:" << st; + return ResultError(st); + } + if (exists) { + LOG(ERROR) << "try to init a directory:" << lfs_index_path << " already exists"; + return ResultError(Status::InternalError("init_fulltext_index directory already exists")); + } + + bool can_use_ram_dir = true; + bool use_compound_file_writer = false; + auto* dir = DorisFSDirectoryFactory::getDirectory(_lfs, lfs_index_path.c_str(), + use_compound_file_writer, can_use_ram_dir, + nullptr, _fs, index_path.c_str()); + _indices_dirs.emplace(std::make_pair(index_id, index_suffix), + std::unique_ptr(dir)); + return dir; +} + +Status InvertedIndexFileWriter::delete_index(const TabletIndex* index_meta) { + if (!index_meta) { + return Status::Error("Index metadata is null."); + } + + auto index_id = index_meta->index_id(); + auto index_suffix = index_meta->get_index_suffix(); + + // Check if the specified index exists + auto index_it = _indices_dirs.find(std::make_pair(index_id, index_suffix)); + if (index_it == _indices_dirs.end()) { + std::ostringstream errMsg; + errMsg << "No inverted index with id " << index_id << " and suffix " << index_suffix + << " found."; + LOG(WARNING) << errMsg.str(); + return Status::OK(); + } + + _indices_dirs.erase(index_it); + return Status::OK(); +} + +size_t InvertedIndexFileWriter::headerLength() { + size_t header_size = 0; + header_size += + sizeof(int) * 2; // Account for the size of the version number and number of indices + for (const auto& entry : _indices_dirs) { + auto suffix = entry.first.second; + header_size += sizeof(int); // index id + header_size += 4; // index suffix name size + header_size += suffix.length(); // index suffix name + header_size += sizeof(int); // index file count + const auto& dir = entry.second; + std::vector files; + dir->list(&files); + + for (auto file : files) { + header_size += 4; // file name size + header_size += file.length(); // file name + header_size += 8; // file offset + header_size += 8; // file size + } + } + return header_size; +} + +Status InvertedIndexFileWriter::close() { + if (_indices_dirs.empty()) { + return Status::OK(); + } + try { + if (_storage_format == InvertedIndexStorageFormatPB::V1) { + for (const auto& entry : _indices_dirs) { + const auto& dir = entry.second; + auto* cfsWriter = _CLNEW DorisCompoundFileWriter(dir.get()); + // write compound file + _file_size += cfsWriter->writeCompoundFile(); + // delete index path, which contains separated inverted index files + if (std::strcmp(dir->getObjectName(), "DorisFSDirectory") == 0) { + auto* compound_dir = static_cast(dir.get()); + compound_dir->deleteDirectory(); + } + _CLDELETE(cfsWriter) + } + } else { + _file_size = write(); + for (const auto& entry : _indices_dirs) { + const auto& dir = entry.second; + // delete index path, which contains separated inverted index files + if (std::strcmp(dir->getObjectName(), "DorisFSDirectory") == 0) { + auto* compound_dir = static_cast(dir.get()); + compound_dir->deleteDirectory(); + } + } + } + } catch (CLuceneError& err) { + return Status::Error( + "CLuceneError occur when close idx file {}, error msg: {}", + InvertedIndexDescriptor::get_index_file_name(_index_file_dir / _segment_file_name), + err.what()); + } + return Status::OK(); +} +size_t InvertedIndexFileWriter::write() { + // Create the output stream to write the compound file + int64_t current_offset = headerLength(); + std::string idx_name = InvertedIndexDescriptor::get_index_file_name(_segment_file_name); + auto* out_dir = DorisFSDirectoryFactory::getDirectory(_fs, _index_file_dir.c_str()); + + auto compound_file_output = + std::unique_ptr(out_dir->createOutput(idx_name.c_str())); + + // Write the version number + compound_file_output->writeInt(InvertedIndexStorageFormatPB::V2); + + // Write the number of indices + const auto numIndices = static_cast(_indices_dirs.size()); + compound_file_output->writeInt(numIndices); + + std::vector> + file_metadata; // Store file name, offset, file length, and corresponding directory + + // First, write all index information and file metadata + for (const auto& entry : _indices_dirs) { + const int64_t index_id = entry.first.first; + const auto& index_suffix = entry.first.second; + const auto& dir = entry.second; + std::vector files; + dir->list(&files); + + auto it = std::find(files.begin(), files.end(), DorisFSDirectory::WRITE_LOCK_FILE); + if (it != files.end()) { + files.erase(it); + } + // sort file list by file length + std::vector> sorted_files; + for (auto file : files) { + sorted_files.emplace_back(file, dir->fileLength(file.c_str())); + } + // TODO: need to optimize + std::sort(sorted_files.begin(), sorted_files.end(), + [](const std::pair& a, + const std::pair& b) { return (a.second < b.second); }); + + int32_t file_count = sorted_files.size(); + + // Write the index ID and the number of files + compound_file_output->writeInt(index_id); + const auto* index_suffix_str = reinterpret_cast(index_suffix.c_str()); + compound_file_output->writeInt(index_suffix.length()); + compound_file_output->writeBytes(index_suffix_str, index_suffix.length()); + compound_file_output->writeInt(file_count); + + // Calculate the offset for each file and write the file metadata + for (const auto& file : sorted_files) { + int64_t file_length = dir->fileLength(file.first.c_str()); + const auto* file_name = reinterpret_cast(file.first.c_str()); + compound_file_output->writeInt(file.first.length()); + compound_file_output->writeBytes(file_name, file.first.length()); + compound_file_output->writeLong(current_offset); + compound_file_output->writeLong(file_length); + + file_metadata.emplace_back(file.first, current_offset, file_length, dir.get()); + current_offset += file_length; // Update the data offset + } + } + + const int64_t buffer_length = 16384; + uint8_t header_buffer[buffer_length]; + + // Next, write the file data + for (const auto& info : file_metadata) { + const std::string& file = std::get<0>(info); + auto* dir = std::get<3>(info); + + // Write the actual file data + DorisCompoundFileWriter::copyFile(file.c_str(), dir, compound_file_output.get(), + header_buffer, buffer_length); + } + + out_dir->close(); + // NOTE: need to decrease ref count, but not to delete here, + // because index cache may get the same directory from DIRECTORIES + _CLDECDELETE(out_dir) + auto compound_file_size = compound_file_output->getFilePointer(); + compound_file_output->close(); + return compound_file_size; +} + +DorisCompoundFileWriter::DorisCompoundFileWriter(CL_NS(store)::Directory* dir) { + if (dir == nullptr) { + _CLTHROWA(CL_ERR_NullPointer, "directory cannot be null"); + } + + directory = dir; +} + +CL_NS(store)::Directory* DorisCompoundFileWriter::getDirectory() { + return directory; +} + +void DorisCompoundFileWriter::sort_files(std::vector& file_infos) { + auto file_priority = [](const std::string& filename) { + if (filename.find("segments") != std::string::npos) return 1; + if (filename.find("fnm") != std::string::npos) return 2; + if (filename.find("tii") != std::string::npos) return 3; + return 4; // Other files + }; + + std::sort(file_infos.begin(), file_infos.end(), [&](const FileInfo& a, const FileInfo& b) { + int32_t priority_a = file_priority(a.filename); + int32_t priority_b = file_priority(b.filename); + if (priority_a != priority_b) return priority_a < priority_b; + return a.filesize < b.filesize; + }); +} + +size_t DorisCompoundFileWriter::writeCompoundFile() { + // list files in current dir + std::vector files; + directory->list(&files); + // remove write.lock file + auto it = std::find(files.begin(), files.end(), DorisFSDirectory::WRITE_LOCK_FILE); + if (it != files.end()) { + files.erase(it); + } + + std::vector sorted_files; + for (auto file : files) { + FileInfo file_info; + file_info.filename = file; + file_info.filesize = directory->fileLength(file.c_str()); + sorted_files.emplace_back(std::move(file_info)); + } + sort_files(sorted_files); + + int32_t file_count = sorted_files.size(); + + io::Path cfs_path(((DorisFSDirectory*)directory)->getCfsDirName()); + auto idx_path = cfs_path.parent_path(); + std::string idx_name = + std::string(cfs_path.stem().c_str()) + DorisFSDirectory::COMPOUND_FILE_EXTENSION; + // write file entries to ram directory to get header length + lucene::store::RAMDirectory ram_dir; + auto* out_idx = ram_dir.createOutput(idx_name.c_str()); + if (out_idx == nullptr) { + LOG(WARNING) << "Write compound file error: RAMDirectory output is nullptr."; + _CLTHROWA(CL_ERR_IO, "Create RAMDirectory output error"); + } + + std::unique_ptr ram_output(out_idx); + ram_output->writeVInt(file_count); + // write file entries in ram directory + // number of files, which data are in header + int header_file_count = 0; + int64_t header_file_length = 0; + const int64_t buffer_length = 16384; + uint8_t ram_buffer[buffer_length]; + for (auto file : sorted_files) { + ram_output->writeString(file.filename); // file name + ram_output->writeLong(0); // data offset + ram_output->writeLong(file.filesize); // file length + header_file_length += file.filesize; + if (header_file_length <= DorisFSDirectory::MAX_HEADER_DATA_SIZE) { + copyFile(file.filename.c_str(), directory, ram_output.get(), ram_buffer, buffer_length); + header_file_count++; + } + } + auto header_len = ram_output->getFilePointer(); + ram_output->close(); + ram_dir.deleteFile(idx_name.c_str()); + ram_dir.close(); + + auto compound_fs = ((DorisFSDirectory*)directory)->getCompoundFileSystem(); + auto* out_dir = DorisFSDirectoryFactory::getDirectory(compound_fs, idx_path.c_str()); + + auto* out = out_dir->createOutput(idx_name.c_str()); + if (out == nullptr) { + LOG(WARNING) << "Write compound file error: CompoundDirectory output is nullptr."; + _CLTHROWA(CL_ERR_IO, "Create CompoundDirectory output error"); + } + std::unique_ptr output(out); + size_t start = output->getFilePointer(); + output->writeVInt(file_count); + // write file entries + int64_t data_offset = header_len; + uint8_t header_buffer[buffer_length]; + for (int i = 0; i < sorted_files.size(); ++i) { + auto file = sorted_files[i]; + output->writeString(file.filename); // FileName + // DataOffset + if (i < header_file_count) { + // file data write in header, so we set its offset to -1. + output->writeLong(-1); + } else { + output->writeLong(data_offset); + } + output->writeLong(file.filesize); // FileLength + if (i < header_file_count) { + // append data + copyFile(file.filename.c_str(), directory, output.get(), header_buffer, buffer_length); + } else { + data_offset += file.filesize; + } + } + // write rest files' data + uint8_t data_buffer[buffer_length]; + for (int i = header_file_count; i < sorted_files.size(); ++i) { + auto file = sorted_files[i]; + copyFile(file.filename.c_str(), directory, output.get(), data_buffer, buffer_length); + } + out_dir->close(); + // NOTE: need to decrease ref count, but not to delete here, + // because index cache may get the same directory from DIRECTORIES + _CLDECDELETE(out_dir) + auto compound_file_size = output->getFilePointer() - start; + output->close(); + //LOG(INFO) << (idx_path / idx_name).c_str() << " size:" << compound_file_size; + return compound_file_size; +} + +void DorisCompoundFileWriter::copyFile(const char* fileName, lucene::store::Directory* dir, + lucene::store::IndexOutput* output, uint8_t* buffer, + int64_t bufferLength) { + lucene::store::IndexInput* tmp = nullptr; + CLuceneError err; + if (!dir->openInput(fileName, tmp, err)) { + throw err; + } + + std::unique_ptr input(tmp); + int64_t start_ptr = output->getFilePointer(); + int64_t length = input->length(); + int64_t remainder = length; + int64_t chunk = bufferLength; + + while (remainder > 0) { + int64_t len = std::min(std::min(chunk, length), remainder); + input->readBytes(buffer, len); + output->writeBytes(buffer, len); + remainder -= len; + } + if (remainder != 0) { + std::ostringstream errMsg; + errMsg << "Non-zero remainder length after copying: " << remainder << " (id: " << fileName + << ", length: " << length << ", buffer size: " << chunk << ")"; + err.set(CL_ERR_IO, errMsg.str().c_str()); + throw err; + } + + int64_t end_ptr = output->getFilePointer(); + int64_t diff = end_ptr - start_ptr; + if (diff != length) { + std::ostringstream errMsg; + errMsg << "Difference in the output file offsets " << diff + << " does not match the original file length " << length; + err.set(CL_ERR_IO, errMsg.str().c_str()); + throw err; + } + input->close(); +} +} // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.h b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.h new file mode 100644 index 0000000000..7e819b0dd7 --- /dev/null +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.h @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include // IWYU pragma: keep +#include +#include + +#include +#include +#include + +#include "io/fs/file_system.h" +#include "olap/rowset/segment_v2/inverted_index_desc.h" + +namespace doris { +class TabletIndex; + +namespace segment_v2 { +class DorisFSDirectory; +using InvertedIndexDirectoryMap = + std::map, std::unique_ptr>; + +class DorisCompoundFileWriter : LUCENE_BASE { +public: + DorisCompoundFileWriter() = default; + DorisCompoundFileWriter(CL_NS(store)::Directory* dir); + ~DorisCompoundFileWriter() override = default; + /** Returns the directory of the compound file. */ + CL_NS(store)::Directory* getDirectory(); + virtual size_t writeCompoundFile(); + static void copyFile(const char* fileName, lucene::store::Directory* dir, + lucene::store::IndexOutput* output, uint8_t* buffer, int64_t bufferLength); + +private: + class FileInfo { + public: + std::string filename; + int32_t filesize; + }; + + void sort_files(std::vector& file_infos); + + CL_NS(store)::Directory* directory = nullptr; +}; + +class InvertedIndexFileWriter { +public: + InvertedIndexFileWriter(io::FileSystemSPtr fs, io::Path segment_file_dir, + std::string segment_file_name, + InvertedIndexStorageFormatPB storage_format) + : _fs(std::move(fs)), + _index_file_dir(std::move(segment_file_dir)), + _segment_file_name(std::move(segment_file_name)), + _storage_format(storage_format) {} + + Result open(const TabletIndex* index_meta); + Status delete_index(const TabletIndex* index_meta); + Status initialize(InvertedIndexDirectoryMap& indices_dirs); + ~InvertedIndexFileWriter() = default; + size_t write(); + Status close(); + size_t headerLength(); + const io::Path& get_index_file_dir() const { return _index_file_dir; }; + std::string get_index_file_name() const { + return InvertedIndexDescriptor::get_index_file_name(_segment_file_name); + }; + std::string get_index_file_path(const TabletIndex* index_meta) const; + size_t get_index_file_size() const { return _file_size; } + const io::FileSystemSPtr& get_fs() const { return _fs; } + +private: + InvertedIndexDirectoryMap _indices_dirs; + const io::FileSystemSPtr _fs; + io::FileSystemSPtr _lfs; + io::Path _index_file_dir; + std::string _segment_file_name; + InvertedIndexStorageFormatPB _storage_format; + size_t _file_size = 0; +}; +} // namespace segment_v2 +} // namespace doris \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp similarity index 58% rename from be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp rename to be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp index 1a0a3e9ab1..05b1dc6f21 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp @@ -15,13 +15,15 @@ // specific language governing permissions and limitations // under the License. -#include "olap/rowset/segment_v2/inverted_index_compound_directory.h" +#include "olap/rowset/segment_v2/inverted_index_fs_directory.h" #include "CLucene/SharedHeader.h" #include "CLucene/_SharedHeader.h" #include "common/status.h" +#include "inverted_index_desc.h" #include "io/fs/file_reader.h" #include "io/fs/file_writer.h" +#include "olap/tablet_schema.h" #include "util/debug_points.h" #include "util/slice.h" @@ -75,184 +77,10 @@ } namespace doris::segment_v2 { -const char* const DorisCompoundDirectory::WRITE_LOCK_FILE = "write.lock"; -const char* const DorisCompoundDirectory::COMPOUND_FILE_EXTENSION = ".idx"; +const char* const DorisFSDirectory::WRITE_LOCK_FILE = "write.lock"; +const char* const DorisFSDirectory::COMPOUND_FILE_EXTENSION = ".idx"; -DorisCompoundFileWriter::DorisCompoundFileWriter(CL_NS(store)::Directory* dir) { - if (dir == nullptr) { - _CLTHROWA(CL_ERR_NullPointer, "directory cannot be null"); - } - - directory = dir; -} - -CL_NS(store)::Directory* DorisCompoundFileWriter::getDirectory() { - return directory; -} - -void DorisCompoundFileWriter::sort_files(std::vector& file_infos) { - auto file_priority = [](const std::string& filename) { - if (filename.find("segments") != std::string::npos) return 1; - if (filename.find("fnm") != std::string::npos) return 2; - if (filename.find("tii") != std::string::npos) return 3; - return 4; // Other files - }; - - std::sort(file_infos.begin(), file_infos.end(), [&](const FileInfo& a, const FileInfo& b) { - int32_t priority_a = file_priority(a.filename); - int32_t priority_b = file_priority(b.filename); - if (priority_a != priority_b) return priority_a < priority_b; - return a.filesize < b.filesize; - }); -} - -size_t DorisCompoundFileWriter::writeCompoundFile() { - // list files in current dir - std::vector files; - directory->list(&files); - // remove write.lock file - auto it = std::find(files.begin(), files.end(), DorisCompoundDirectory::WRITE_LOCK_FILE); - if (it != files.end()) { - files.erase(it); - } - - std::vector sorted_files; - for (auto file : files) { - FileInfo file_info; - file_info.filename = file; - file_info.filesize = directory->fileLength(file.c_str()); - sorted_files.emplace_back(std::move(file_info)); - } - sort_files(sorted_files); - - int32_t file_count = sorted_files.size(); - - io::Path cfs_path(((DorisCompoundDirectory*)directory)->getCfsDirName()); - auto idx_path = cfs_path.parent_path(); - std::string idx_name = - std::string(cfs_path.stem().c_str()) + DorisCompoundDirectory::COMPOUND_FILE_EXTENSION; - // write file entries to ram directory to get header length - lucene::store::RAMDirectory ram_dir; - auto* out_idx = ram_dir.createOutput(idx_name.c_str()); - if (out_idx == nullptr) { - LOG(WARNING) << "Write compound file error: RAMDirectory output is nullptr."; - _CLTHROWA(CL_ERR_IO, "Create RAMDirectory output error"); - } - - std::unique_ptr ram_output(out_idx); - ram_output->writeVInt(file_count); - // write file entries in ram directory - // number of files, which data are in header - int header_file_count = 0; - int64_t header_file_length = 0; - const int64_t buffer_length = 16384; - uint8_t ram_buffer[buffer_length]; - for (auto file : sorted_files) { - ram_output->writeString(file.filename); // file name - ram_output->writeLong(0); // data offset - ram_output->writeLong(file.filesize); // file length - header_file_length += file.filesize; - if (header_file_length <= DorisCompoundDirectory::MAX_HEADER_DATA_SIZE) { - copyFile(file.filename.c_str(), ram_output.get(), ram_buffer, buffer_length); - header_file_count++; - } - } - auto header_len = ram_output->getFilePointer(); - ram_output->close(); - ram_dir.deleteFile(idx_name.c_str()); - ram_dir.close(); - - auto compound_fs = ((DorisCompoundDirectory*)directory)->getCompoundFileSystem(); - auto* out_dir = DorisCompoundDirectoryFactory::getDirectory(compound_fs, idx_path.c_str()); - - auto* out = out_dir->createOutput(idx_name.c_str()); - if (out == nullptr) { - LOG(WARNING) << "Write compound file error: CompoundDirectory output is nullptr."; - _CLTHROWA(CL_ERR_IO, "Create CompoundDirectory output error"); - } - std::unique_ptr output(out); - size_t start = output->getFilePointer(); - output->writeVInt(file_count); - // write file entries - int64_t data_offset = header_len; - uint8_t header_buffer[buffer_length]; - for (int i = 0; i < sorted_files.size(); ++i) { - auto file = sorted_files[i]; - output->writeString(file.filename); // FileName - // DataOffset - if (i < header_file_count) { - // file data write in header, so we set its offset to -1. - output->writeLong(-1); - } else { - output->writeLong(data_offset); - } - output->writeLong(file.filesize); // FileLength - if (i < header_file_count) { - // append data - copyFile(file.filename.c_str(), output.get(), header_buffer, buffer_length); - } else { - data_offset += file.filesize; - } - } - // write rest files' data - uint8_t data_buffer[buffer_length]; - for (int i = header_file_count; i < sorted_files.size(); ++i) { - auto file = sorted_files[i]; - copyFile(file.filename.c_str(), output.get(), data_buffer, buffer_length); - } - out_dir->close(); - // NOTE: need to decrease ref count, but not to delete here, - // because index cache may get the same directory from DIRECTORIES - _CLDECDELETE(out_dir) - auto compound_file_size = output->getFilePointer() - start; - output->close(); - //LOG(INFO) << (idx_path / idx_name).c_str() << " size:" << compound_file_size; - return compound_file_size; -} - -void DorisCompoundFileWriter::copyFile(const char* fileName, lucene::store::IndexOutput* output, - uint8_t* buffer, int64_t bufferLength) { - lucene::store::IndexInput* tmp = nullptr; - CLuceneError err; - if (!directory->openInput(fileName, tmp, err)) { - throw err; - } - - std::unique_ptr input(tmp); - int64_t start_ptr = output->getFilePointer(); - int64_t length = input->length(); - int64_t remainder = length; - int64_t chunk = bufferLength; - - while (remainder > 0) { - int64_t len = std::min(std::min(chunk, length), remainder); - input->readBytes(buffer, len); - output->writeBytes(buffer, len); - remainder -= len; - } - if (remainder != 0) { - TCHAR buf[CL_MAX_PATH + 100]; - swprintf(buf, CL_MAX_PATH + 100, - _T("Non-zero remainder length after copying") - _T(": %d (id: %s, length: %d, buffer size: %d)"), - (int)remainder, fileName, (int)length, (int)chunk); - _CLTHROWT(CL_ERR_IO, buf); - } - - int64_t end_ptr = output->getFilePointer(); - int64_t diff = end_ptr - start_ptr; - if (diff != length) { - TCHAR buf[100]; - swprintf(buf, 100, - _T("Difference in the output file offsets %d ") - _T("does not match the original file length %d"), - (int)diff, (int)length); - _CLTHROWA(CL_ERR_IO, buf); - } - input->close(); -} - -class DorisCompoundDirectory::FSIndexOutput : public lucene::store::BufferedIndexOutput { +class DorisFSDirectory::FSIndexOutput : public lucene::store::BufferedIndexOutput { private: io::FileWriterPtr _writer; @@ -267,15 +95,15 @@ public: int64_t length() const override; }; -bool DorisCompoundDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, const char* path, - IndexInput*& ret, CLuceneError& error, - int32_t buffer_size) { +bool DorisFSDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, const char* path, + IndexInput*& ret, CLuceneError& error, + int32_t buffer_size) { CND_PRECONDITION(path != nullptr, "path is NULL"); if (buffer_size == -1) { buffer_size = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE; } - auto* h = _CLNEW SharedHandle(path); + auto h = std::make_shared(path); io::FileReaderOptions reader_options; reader_options.cache_type = config::enable_file_cache ? io::FileCachePolicy::FILE_BLOCK_CACHE @@ -290,7 +118,7 @@ bool DorisCompoundDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, co //Store the file length h->_length = h->_reader->size(); h->_fpos = 0; - ret = _CLNEW FSIndexInput(h, buffer_size); + ret = _CLNEW FSIndexInput(std::move(h), buffer_size); return true; } else { @@ -305,31 +133,31 @@ bool DorisCompoundDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, co error.set(CL_ERR_IO, "Could not open file"); } } - delete h->_shared_lock; - _CLDECDELETE(h) + //delete h->_shared_lock; + //_CLDECDELETE(h) return false; } -DorisCompoundDirectory::FSIndexInput::FSIndexInput(const FSIndexInput& other) +DorisFSDirectory::FSIndexInput::FSIndexInput(const FSIndexInput& other) : BufferedIndexInput(other) { if (other._handle == nullptr) { _CLTHROWA(CL_ERR_NullPointer, "other handle is null"); } - std::lock_guard wlock(*other._handle->_shared_lock); - _handle = _CL_POINTER(other._handle); + std::lock_guard wlock(other._handle->_shared_lock); + _handle = other._handle; _pos = other._handle->_fpos; //note where we are currently... _io_ctx = other._io_ctx; } -DorisCompoundDirectory::FSIndexInput::SharedHandle::SharedHandle(const char* path) { +DorisFSDirectory::FSIndexInput::SharedHandle::SharedHandle(const char* path) { _length = 0; _fpos = 0; strcpy(this->path, path); - _shared_lock = new std::mutex(); + //_shared_lock = new std::mutex(); } -DorisCompoundDirectory::FSIndexInput::SharedHandle::~SharedHandle() { +DorisFSDirectory::FSIndexInput::SharedHandle::~SharedHandle() { if (_reader) { if (_reader->close().ok()) { _reader = nullptr; @@ -337,16 +165,16 @@ DorisCompoundDirectory::FSIndexInput::SharedHandle::~SharedHandle() { } } -DorisCompoundDirectory::FSIndexInput::~FSIndexInput() { +DorisFSDirectory::FSIndexInput::~FSIndexInput() { FSIndexInput::close(); } -lucene::store::IndexInput* DorisCompoundDirectory::FSIndexInput::clone() const { - return _CLNEW DorisCompoundDirectory::FSIndexInput(*this); +lucene::store::IndexInput* DorisFSDirectory::FSIndexInput::clone() const { + return _CLNEW DorisFSDirectory::FSIndexInput(*this); } -void DorisCompoundDirectory::FSIndexInput::close() { +void DorisFSDirectory::FSIndexInput::close() { BufferedIndexInput::close(); - if (_handle != nullptr) { + /*if (_handle != nullptr) { std::mutex* lock = _handle->_shared_lock; bool ref = false; { @@ -361,19 +189,19 @@ void DorisCompoundDirectory::FSIndexInput::close() { if (!ref) { delete lock; } - } + }*/ } -void DorisCompoundDirectory::FSIndexInput::seekInternal(const int64_t position) { +void DorisFSDirectory::FSIndexInput::seekInternal(const int64_t position) { CND_PRECONDITION(position >= 0 && position < _handle->_length, "Seeking out of range"); _pos = position; } /** IndexInput methods */ -void DorisCompoundDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len) { +void DorisFSDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len) { CND_PRECONDITION(_handle != nullptr, "shared file handle has closed"); CND_PRECONDITION(_handle->_reader != nullptr, "file is not open"); - std::lock_guard wlock(*_handle->_shared_lock); + std::lock_guard wlock(_handle->_shared_lock); int64_t position = getFilePointer(); if (_pos != position) { @@ -397,12 +225,11 @@ void DorisCompoundDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_ _handle->_fpos = _pos; } -void DorisCompoundDirectory::FSIndexOutput::init(const io::FileSystemSPtr& fileSystem, - const char* path) { +void DorisFSDirectory::FSIndexOutput::init(const io::FileSystemSPtr& fileSystem, const char* path) { io::FileWriterOptions opts {.create_empty_file = false}; Status status = fileSystem->create_file(path, &_writer, &opts); DBUG_EXECUTE_IF( - "DorisCompoundDirectory::FSIndexOutput._throw_clucene_error_in_fsindexoutput_" + "DorisFSDirectory::FSIndexOutput._throw_clucene_error_in_fsindexoutput_" "init", { status = Status::Error( @@ -416,12 +243,12 @@ void DorisCompoundDirectory::FSIndexOutput::init(const io::FileSystemSPtr& fileS } } -DorisCompoundDirectory::FSIndexOutput::~FSIndexOutput() { +DorisFSDirectory::FSIndexOutput::~FSIndexOutput() { if (_writer) { try { FSIndexOutput::close(); DBUG_EXECUTE_IF( - "DorisCompoundDirectory::FSIndexOutput._throw_clucene_error_in_fsindexoutput_" + "DorisFSDirectory::FSIndexOutput._throw_clucene_error_in_fsindexoutput_" "destructor", { _CLTHROWA(CL_ERR_IO, @@ -434,11 +261,11 @@ DorisCompoundDirectory::FSIndexOutput::~FSIndexOutput() { } } -void DorisCompoundDirectory::FSIndexOutput::flushBuffer(const uint8_t* b, const int32_t size) { +void DorisFSDirectory::FSIndexOutput::flushBuffer(const uint8_t* b, const int32_t size) { if (_writer != nullptr && b != nullptr && size > 0) { Slice data {b, (size_t)size}; DBUG_EXECUTE_IF( - "DorisCompoundDirectory::FSIndexOutput._mock_append_data_error_in_fsindexoutput_" + "DorisFSDirectory::FSIndexOutput._mock_append_data_error_in_fsindexoutput_" "flushBuffer", { if (_writer->path().filename() == "_0.tii" || @@ -448,8 +275,7 @@ void DorisCompoundDirectory::FSIndexOutput::flushBuffer(const uint8_t* b, const }) Status st = _writer->append(data); DBUG_EXECUTE_IF( - "DorisCompoundDirectory::FSIndexOutput._status_error_in_fsindexoutput_flushBuffer", - { + "DorisFSDirectory::FSIndexOutput._status_error_in_fsindexoutput_flushBuffer", { st = Status::Error( "flush buffer mock error"); }) @@ -459,20 +285,20 @@ void DorisCompoundDirectory::FSIndexOutput::flushBuffer(const uint8_t* b, const } } else { if (_writer == nullptr) { - LOG(WARNING) << "File writer is nullptr in DorisCompoundDirectory::FSIndexOutput, " + LOG(WARNING) << "File writer is nullptr in DorisFSDirectory::FSIndexOutput, " "ignore flush."; } else if (b == nullptr) { LOG(WARNING) << "buffer is nullptr when flushBuffer in " - "DorisCompoundDirectory::FSIndexOutput"; + "DorisFSDirectory::FSIndexOutput"; } } } -void DorisCompoundDirectory::FSIndexOutput::close() { +void DorisFSDirectory::FSIndexOutput::close() { try { BufferedIndexOutput::close(); DBUG_EXECUTE_IF( - "DorisCompoundDirectory::FSIndexOutput._throw_clucene_error_in_bufferedindexoutput_" + "DorisFSDirectory::FSIndexOutput._throw_clucene_error_in_bufferedindexoutput_" "close", { _CLTHROWA(CL_ERR_IO, @@ -489,7 +315,7 @@ void DorisCompoundDirectory::FSIndexOutput::close() { } if (_writer) { Status ret = _writer->finalize(); - DBUG_EXECUTE_IF("DorisCompoundDirectory::FSIndexOutput._set_writer_finalize_status_error", + DBUG_EXECUTE_IF("DorisFSDirectory::FSIndexOutput._set_writer_finalize_status_error", { ret = Status::Error("writer finalize status error"); }) if (!ret.ok()) { LOG(WARNING) << "FSIndexOutput close, file writer finalize error: " << ret.to_string(); @@ -497,7 +323,7 @@ void DorisCompoundDirectory::FSIndexOutput::close() { _CLTHROWA(CL_ERR_IO, ret.to_string().c_str()); } ret = _writer->close(); - DBUG_EXECUTE_IF("DorisCompoundDirectory::FSIndexOutput._set_writer_close_status_error", + DBUG_EXECUTE_IF("DorisFSDirectory::FSIndexOutput._set_writer_close_status_error", { ret = Status::Error("writer close status error"); }) if (!ret.ok()) { LOG(WARNING) << "FSIndexOutput close, file writer close error: " << ret.to_string(); @@ -510,7 +336,7 @@ void DorisCompoundDirectory::FSIndexOutput::close() { _writer.reset(nullptr); } -int64_t DorisCompoundDirectory::FSIndexOutput::length() const { +int64_t DorisFSDirectory::FSIndexOutput::length() const { CND_PRECONDITION(_writer != nullptr, "file is not open"); int64_t ret; if (!_writer->fs()->file_size(_writer->path(), &ret).ok()) { @@ -519,15 +345,14 @@ int64_t DorisCompoundDirectory::FSIndexOutput::length() const { return ret; } -DorisCompoundDirectory::DorisCompoundDirectory() { +DorisFSDirectory::DorisFSDirectory() { filemode = 0644; this->lockFactory = nullptr; } -void DorisCompoundDirectory::init(const io::FileSystemSPtr& _fs, const char* _path, - bool use_compound_file_writer, - lucene::store::LockFactory* lock_factory, - const io::FileSystemSPtr& cfs, const char* cfs_path) { +void DorisFSDirectory::init(const io::FileSystemSPtr& _fs, const char* _path, + bool use_compound_file_writer, lucene::store::LockFactory* lock_factory, + const io::FileSystemSPtr& cfs, const char* cfs_path) { fs = _fs; directory = _path; useCompoundFileWriter = use_compound_file_writer; @@ -562,23 +387,23 @@ void DorisCompoundDirectory::init(const io::FileSystemSPtr& _fs, const char* _pa } } -void DorisCompoundDirectory::priv_getFN(char* buffer, const char* name) const { +void DorisFSDirectory::priv_getFN(char* buffer, const char* name) const { buffer[0] = 0; strcpy(buffer, directory.c_str()); strcat(buffer, PATH_DELIMITERA); strcat(buffer, name); } -DorisCompoundDirectory::~DorisCompoundDirectory() = default; +DorisFSDirectory::~DorisFSDirectory() = default; -const char* DorisCompoundDirectory::getClassName() { - return "DorisCompoundDirectory"; +const char* DorisFSDirectory::getClassName() { + return "DorisFSDirectory"; } -const char* DorisCompoundDirectory::getObjectName() const { +const char* DorisFSDirectory::getObjectName() const { return getClassName(); } -bool DorisCompoundDirectory::list(std::vector* names) const { +bool DorisFSDirectory::list(std::vector* names) const { CND_PRECONDITION(!directory.empty(), "directory is not open"); char fl[CL_MAX_DIR]; priv_getFN(fl, ""); @@ -591,7 +416,7 @@ bool DorisCompoundDirectory::list(std::vector* names) const { return true; } -bool DorisCompoundDirectory::fileExists(const char* name) const { +bool DorisFSDirectory::fileExists(const char* name) const { CND_PRECONDITION(directory[0] != 0, "directory is not open"); char fl[CL_MAX_DIR]; priv_getFN(fl, name); @@ -600,11 +425,15 @@ bool DorisCompoundDirectory::fileExists(const char* name) const { return exists; } -const char* DorisCompoundDirectory::getCfsDirName() const { +const char* DorisFSDirectory::getCfsDirName() const { return cfs_directory.c_str(); } -int64_t DorisCompoundDirectory::fileModified(const char* name) const { +const std::string& DorisFSDirectory::getDirName() const { + return directory; +} + +int64_t DorisFSDirectory::fileModified(const char* name) const { CND_PRECONDITION(directory[0] != 0, "directory is not open"); struct stat buf; char buffer[CL_MAX_DIR]; @@ -616,7 +445,7 @@ int64_t DorisCompoundDirectory::fileModified(const char* name) const { } } -void DorisCompoundDirectory::touchFile(const char* name) { +void DorisFSDirectory::touchFile(const char* name) { CND_PRECONDITION(directory[0] != 0, "directory is not open"); char buffer[CL_MAX_DIR]; snprintf(buffer, CL_MAX_DIR, "%s%s%s", directory.c_str(), PATH_DELIMITERA, name); @@ -625,7 +454,7 @@ void DorisCompoundDirectory::touchFile(const char* name) { LOG_AND_THROW_IF_ERROR(fs->create_file(buffer, &tmp_writer), "Touch file IO error") } -int64_t DorisCompoundDirectory::fileLength(const char* name) const { +int64_t DorisFSDirectory::fileLength(const char* name) const { CND_PRECONDITION(directory[0] != 0, "directory is not open"); char buffer[CL_MAX_DIR]; priv_getFN(buffer, name); @@ -634,26 +463,17 @@ int64_t DorisCompoundDirectory::fileLength(const char* name) const { return size; } -bool DorisCompoundDirectory::openInput(const char* name, lucene::store::IndexInput*& ret, - CLuceneError& error, int32_t bufferSize) { +bool DorisFSDirectory::openInput(const char* name, lucene::store::IndexInput*& ret, + CLuceneError& error, int32_t bufferSize) { CND_PRECONDITION(directory[0] != 0, "directory is not open"); char fl[CL_MAX_DIR]; priv_getFN(fl, name); return FSIndexInput::open(fs, fl, ret, error, bufferSize); } -void DorisCompoundDirectory::close() { - if (useCompoundFileWriter) { - auto* cfsWriter = _CLNEW DorisCompoundFileWriter(this); - // write compound file - compound_file_size = cfsWriter->writeCompoundFile(); - // delete index path, which contains separated inverted index files - deleteDirectory(); - _CLDELETE(cfsWriter) - } -} +void DorisFSDirectory::close() {} -bool DorisCompoundDirectory::doDeleteFile(const char* name) { +bool DorisFSDirectory::doDeleteFile(const char* name) { CND_PRECONDITION(directory[0] != 0, "directory is not open"); char fl[CL_MAX_DIR]; priv_getFN(fl, name); @@ -661,7 +481,7 @@ bool DorisCompoundDirectory::doDeleteFile(const char* name) { return true; } -bool DorisCompoundDirectory::deleteDirectory() { +bool DorisFSDirectory::deleteDirectory() { CND_PRECONDITION(directory[0] != 0, "directory is not open"); char fl[CL_MAX_DIR]; priv_getFN(fl, ""); @@ -670,7 +490,7 @@ bool DorisCompoundDirectory::deleteDirectory() { return true; } -void DorisCompoundDirectory::renameFile(const char* from, const char* to) { +void DorisFSDirectory::renameFile(const char* from, const char* to) { CND_PRECONDITION(directory[0] != 0, "directory is not open"); std::lock_guard wlock(_this_lock); char old[CL_MAX_DIR]; @@ -687,7 +507,7 @@ void DorisCompoundDirectory::renameFile(const char* from, const char* to) { LOG_AND_THROW_IF_ERROR(fs->rename(old, nu), fmt::format("Rename {} to {} IO error", old, nu)) } -lucene::store::IndexOutput* DorisCompoundDirectory::createOutput(const char* name) { +lucene::store::IndexOutput* DorisFSDirectory::createOutput(const char* name) { CND_PRECONDITION(directory[0] != 0, "directory is not open"); char fl[CL_MAX_DIR]; priv_getFN(fl, name); @@ -711,27 +531,26 @@ lucene::store::IndexOutput* DorisCompoundDirectory::createOutput(const char* nam return ret; } -std::string DorisCompoundDirectory::toString() const { - return std::string("DorisCompoundDirectory@") + this->directory; +std::string DorisFSDirectory::toString() const { + return std::string("DorisFSDirectory@") + this->directory; } -/** - * DorisRAMCompoundDirectory - */ -DorisRAMCompoundDirectory::DorisRAMCompoundDirectory() { +DorisRAMFSDirectory::DorisRAMFSDirectory() { filesMap = _CLNEW FileMap(true, true); this->sizeInBytes = 0; } -DorisRAMCompoundDirectory::~DorisRAMCompoundDirectory() { +DorisRAMFSDirectory::~DorisRAMFSDirectory() { + std::lock_guard wlock(_this_lock); + filesMap->clear(); _CLDELETE(lockFactory); _CLDELETE(filesMap); } -void DorisRAMCompoundDirectory::init(const io::FileSystemSPtr& _fs, const char* _path, - bool use_compound_file_writer, - lucene::store::LockFactory* lock_factory, - const io::FileSystemSPtr& cfs, const char* cfs_path) { +void DorisRAMFSDirectory::init(const io::FileSystemSPtr& _fs, const char* _path, + bool use_compound_file_writer, + lucene::store::LockFactory* lock_factory, + const io::FileSystemSPtr& cfs, const char* cfs_path) { fs = _fs; directory = _path; useCompoundFileWriter = use_compound_file_writer; @@ -750,7 +569,7 @@ void DorisRAMCompoundDirectory::init(const io::FileSystemSPtr& _fs, const char* lucene::store::Directory::setLockFactory(_CLNEW lucene::store::SingleInstanceLockFactory()); } -bool DorisRAMCompoundDirectory::list(std::vector* names) const { +bool DorisRAMFSDirectory::list(std::vector* names) const { std::lock_guard wlock(_this_lock); auto itr = filesMap->begin(); while (itr != filesMap->end()) { @@ -760,18 +579,18 @@ bool DorisRAMCompoundDirectory::list(std::vector* names) const { return true; } -bool DorisRAMCompoundDirectory::fileExists(const char* name) const { +bool DorisRAMFSDirectory::fileExists(const char* name) const { std::lock_guard wlock(_this_lock); return filesMap->exists((char*)name); } -int64_t DorisRAMCompoundDirectory::fileModified(const char* name) const { +int64_t DorisRAMFSDirectory::fileModified(const char* name) const { std::lock_guard wlock(_this_lock); auto* f = filesMap->get((char*)name); return f->getLastModified(); } -void DorisRAMCompoundDirectory::touchFile(const char* name) { +void DorisRAMFSDirectory::touchFile(const char* name) { lucene::store::RAMFile* file = nullptr; { std::lock_guard wlock(_this_lock); @@ -789,14 +608,14 @@ void DorisRAMCompoundDirectory::touchFile(const char* name) { file->setLastModified(ts2); } -int64_t DorisRAMCompoundDirectory::fileLength(const char* name) const { +int64_t DorisRAMFSDirectory::fileLength(const char* name) const { std::lock_guard wlock(_this_lock); auto* f = filesMap->get((char*)name); return f->getLength(); } -bool DorisRAMCompoundDirectory::openInput(const char* name, lucene::store::IndexInput*& ret, - CLuceneError& error, int32_t bufferSize) { +bool DorisRAMFSDirectory::openInput(const char* name, lucene::store::IndexInput*& ret, + CLuceneError& error, int32_t bufferSize) { std::lock_guard wlock(_this_lock); auto* file = filesMap->get((char*)name); if (file == nullptr) { @@ -808,16 +627,11 @@ bool DorisRAMCompoundDirectory::openInput(const char* name, lucene::store::Index return true; } -void DorisRAMCompoundDirectory::close() { - // write compound file - DorisCompoundDirectory::close(); - - std::lock_guard wlock(_this_lock); - filesMap->clear(); - _CLDELETE(filesMap); +void DorisRAMFSDirectory::close() { + DorisFSDirectory::close(); } -bool DorisRAMCompoundDirectory::doDeleteFile(const char* name) { +bool DorisRAMFSDirectory::doDeleteFile(const char* name) { std::lock_guard wlock(_this_lock); auto itr = filesMap->find((char*)name); if (itr != filesMap->end()) { @@ -830,12 +644,12 @@ bool DorisRAMCompoundDirectory::doDeleteFile(const char* name) { } } -bool DorisRAMCompoundDirectory::deleteDirectory() { +bool DorisRAMFSDirectory::deleteDirectory() { // do nothing, RAM dir do not have actual files return true; } -void DorisRAMCompoundDirectory::renameFile(const char* from, const char* to) { +void DorisRAMFSDirectory::renameFile(const char* from, const char* to) { std::lock_guard wlock(_this_lock); auto itr = filesMap->find((char*)from); @@ -861,7 +675,7 @@ void DorisRAMCompoundDirectory::renameFile(const char* from, const char* to) { filesMap->put(strdup(to), file); } -lucene::store::IndexOutput* DorisRAMCompoundDirectory::createOutput(const char* name) { +lucene::store::IndexOutput* DorisRAMFSDirectory::createOutput(const char* name) { /* Check the $filesMap VoidMap to see if there was a previous file named ** $name. If so, delete the old RAMFile object, but reuse the existing ** char buffer ($n) that holds the filename. If not, duplicate the @@ -888,22 +702,19 @@ lucene::store::IndexOutput* DorisRAMCompoundDirectory::createOutput(const char* return _CLNEW lucene::store::RAMOutputStream(file); } -std::string DorisRAMCompoundDirectory::toString() const { - return std::string("DorisRAMCompoundDirectory@") + this->directory; +std::string DorisRAMFSDirectory::toString() const { + return std::string("DorisRAMFSDirectory@") + this->directory; } -const char* DorisRAMCompoundDirectory::getClassName() { - return "DorisRAMCompoundDirectory"; +const char* DorisRAMFSDirectory::getClassName() { + return "DorisRAMFSDirectory"; } -const char* DorisRAMCompoundDirectory::getObjectName() const { +const char* DorisRAMFSDirectory::getObjectName() const { return getClassName(); } -/** - * DorisCompoundDirectoryFactory - */ -DorisCompoundDirectory* DorisCompoundDirectoryFactory::getDirectory( +DorisFSDirectory* DorisFSDirectoryFactory::getDirectory( const io::FileSystemSPtr& _fs, const char* _file, bool use_compound_file_writer, bool can_use_ram_dir, lucene::store::LockFactory* lock_factory, const io::FileSystemSPtr& _cfs, const char* _cfs_file) { @@ -911,7 +722,7 @@ DorisCompoundDirectory* DorisCompoundDirectoryFactory::getDirectory( if (cfs_file == nullptr) { cfs_file = _file; } - DorisCompoundDirectory* dir = nullptr; + DorisFSDirectory* dir = nullptr; if (!_file || !*_file) { _CLTHROWA(CL_ERR_IO, "Invalid directory"); } @@ -922,7 +733,7 @@ DorisCompoundDirectory* DorisCompoundDirectoryFactory::getDirectory( // 1. only write separated index files, which is can_use_ram_dir = true. // 2. config::inverted_index_ram_dir_enable = true if (config::inverted_index_ram_dir_enable && can_use_ram_dir) { - dir = _CLNEW DorisRAMCompoundDirectory(); + dir = _CLNEW DorisRAMFSDirectory(); } else { bool exists = false; LOG_AND_THROW_IF_ERROR(_fs->exists(file, &exists), "Get directory exists IO error") @@ -930,7 +741,7 @@ DorisCompoundDirectory* DorisCompoundDirectoryFactory::getDirectory( LOG_AND_THROW_IF_ERROR(_fs->create_directory(file), "Get directory create directory IO error") } - dir = _CLNEW DorisCompoundDirectory(); + dir = _CLNEW DorisFSDirectory(); } dir->init(_fs, file, use_compound_file_writer, lock_factory, _cfs, cfs_file); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h similarity index 76% rename from be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h rename to be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h index 4c238c1380..0478070b4f 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h @@ -38,31 +38,12 @@ namespace lucene::store { class LockFactory; } // namespace lucene::store -namespace doris::segment_v2 { +namespace doris { +class TabletIndex; -class DorisCompoundFileWriter : LUCENE_BASE { -public: - DorisCompoundFileWriter(CL_NS(store)::Directory* dir); - ~DorisCompoundFileWriter() override = default; - /** Returns the directory of the compound file. */ - CL_NS(store)::Directory* getDirectory(); - size_t writeCompoundFile(); - void copyFile(const char* fileName, lucene::store::IndexOutput* output, uint8_t* buffer, - int64_t bufferLength); +namespace segment_v2 { -private: - class FileInfo { - public: - std::string filename; - int32_t filesize; - }; - - void sort_files(std::vector& file_infos); - - CL_NS(store)::Directory* directory = nullptr; -}; - -class CLUCENE_EXPORT DorisCompoundDirectory : public lucene::store::Directory { +class CLUCENE_EXPORT DorisFSDirectory : public lucene::store::Directory { public: static const char* const WRITE_LOCK_FILE; static const char* const COMPOUND_FILE_EXTENSION; @@ -87,17 +68,18 @@ public: class FSIndexOutput; class FSIndexInput; - friend class DorisCompoundDirectory::FSIndexOutput; - friend class DorisCompoundDirectory::FSIndexInput; + friend class DorisFSDirectory::FSIndexOutput; + friend class DorisFSDirectory::FSIndexInput; const io::FileSystemSPtr& getFileSystem() { return fs; } const io::FileSystemSPtr& getCompoundFileSystem() { return compound_fs; } size_t getCompoundFileSize() const { return compound_file_size; } - ~DorisCompoundDirectory() override; + ~DorisFSDirectory() override; bool list(std::vector* names) const override; bool fileExists(const char* name) const override; const char* getCfsDirName() const; + const std::string& getDirName() const; int64_t fileModified(const char* name) const override; int64_t fileLength(const char* name) const override; bool openInput(const char* name, lucene::store::IndexInput*& ret, CLuceneError& err, @@ -111,7 +93,7 @@ public: const char* getObjectName() const override; virtual bool deleteDirectory(); - DorisCompoundDirectory(); + DorisFSDirectory(); virtual void init(const io::FileSystemSPtr& fs, const char* path, bool use_compound_file_writer, lucene::store::LockFactory* lock_factory = nullptr, @@ -119,7 +101,7 @@ public: const char* cfs_path = nullptr); }; -class CLUCENE_EXPORT DorisRAMCompoundDirectory : public DorisCompoundDirectory { +class CLUCENE_EXPORT DorisRAMFSDirectory : public DorisFSDirectory { protected: using FileMap = lucene::util::CLHashMap* names) const override; /** Constructs an empty {@link Directory}. */ - DorisRAMCompoundDirectory(); + DorisRAMFSDirectory(); ///Destructor - only call this if you are sure the directory ///is not being used anymore. Otherwise use the ref-counting ///facilities of dir->close - ~DorisRAMCompoundDirectory() override; + ~DorisRAMFSDirectory() override; bool doDeleteFile(const char* name) override; @@ -182,25 +164,27 @@ public: const char* getObjectName() const override; }; -class DorisCompoundDirectory::FSIndexInput : public lucene::store::BufferedIndexInput { +class DorisFSDirectory::FSIndexInput : public lucene::store::BufferedIndexInput { class SharedHandle : LUCENE_REFBASE { public: io::FileReaderSPtr _reader; uint64_t _length; int64_t _fpos; - std::mutex* _shared_lock = nullptr; + std::mutex _shared_lock; + //std::mutex* _shared_lock = nullptr; char path[4096]; SharedHandle(const char* path); ~SharedHandle() override; }; - SharedHandle* _handle = nullptr; + std::shared_ptr _handle = nullptr; int64_t _pos; io::IOContext _io_ctx; - FSIndexInput(SharedHandle* handle, int32_t buffer_size) : BufferedIndexInput(buffer_size) { + FSIndexInput(std::shared_ptr handle, int32_t buffer_size) + : BufferedIndexInput(buffer_size) { this->_pos = 0; - this->_handle = handle; + this->_handle = std::move(handle); this->_io_ctx.reader_type = ReaderType::READER_QUERY; this->_io_ctx.is_index_data = false; } @@ -217,7 +201,7 @@ public: void close() override; int64_t length() const override { return _handle->_length; } - const char* getDirectoryType() const override { return DorisCompoundDirectory::getClassName(); } + const char* getDirectoryType() const override { return DorisFSDirectory::getClassName(); } const char* getObjectName() const override { return getClassName(); } static const char* getClassName() { return "FSIndexInput"; } @@ -233,15 +217,16 @@ protected: }; /** - * Factory function to create DorisCompoundDirectory + * Factory function to create DorisFSDirectory */ -class DorisCompoundDirectoryFactory { +class DorisFSDirectoryFactory { public: - static DorisCompoundDirectory* getDirectory(const io::FileSystemSPtr& fs, const char* file, - bool use_compound_file_writer = false, - bool can_use_ram_dir = false, - lucene::store::LockFactory* lock_factory = nullptr, - const io::FileSystemSPtr& cfs_fs = nullptr, - const char* cfs_file = nullptr); + static DorisFSDirectory* getDirectory(const io::FileSystemSPtr& fs, const char* file, + bool use_compound_file_writer = false, + bool can_use_ram_dir = false, + lucene::store::LockFactory* lock_factory = nullptr, + const io::FileSystemSPtr& cfs_fs = nullptr, + const char* cfs_file = nullptr); }; -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace segment_v2 +} // namespace doris \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index c83158ed26..58810d472e 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -33,7 +33,6 @@ #include #include #include -#include #include #include @@ -60,7 +59,8 @@ #include "olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h" #include "olap/rowset/segment_v2/inverted_index/query/query_factory.h" #include "olap/rowset/segment_v2/inverted_index_cache.h" -#include "olap/rowset/segment_v2/inverted_index_compound_directory.h" +#include "olap/rowset/segment_v2/inverted_index_file_reader.h" +#include "olap/rowset/segment_v2/inverted_index_fs_directory.h" #include "olap/rowset/segment_v2/inverted_index_searcher.h" #include "olap/types.h" #include "runtime/runtime_state.h" @@ -149,22 +149,19 @@ Status InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cach bool owned_dir = false; try { // try to get query bitmap result from cache and return immediately on cache hit - io::Path path(_path); - auto index_dir = path.parent_path(); - auto index_file_name = InvertedIndexDescriptor::get_index_file_name( - path.filename(), _index_meta.index_id(), _index_meta.get_index_suffix()); - auto index_file_path = index_dir / index_file_name; + auto index_file_path = _inverted_index_file_reader->get_index_file_path(&_index_meta); InvertedIndexQueryCache::CacheKey cache_key { index_file_path, "", InvertedIndexQueryType::UNKNOWN_QUERY, "null_bitmap"}; auto* cache = InvertedIndexQueryCache::instance(); if (cache->lookup(cache_key, cache_handle)) { return Status::OK(); } + std::unique_ptr multi_compound_reader; if (!dir) { - dir = new DorisCompoundReader( - DorisCompoundDirectoryFactory::getDirectory(_fs, index_dir.c_str()), - index_file_name.c_str(), config::inverted_index_read_buffer_size); + // TODO: ugly code here, try to refact. + auto directory = DORIS_TRY(_inverted_index_file_reader->open(&_index_meta)); + dir = directory.release(); owned_dir = true; } @@ -199,8 +196,8 @@ Status InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cach Status InvertedIndexReader::handle_searcher_cache( InvertedIndexCacheHandle* inverted_index_cache_handle, OlapReaderStatistics* stats) { - auto index_file_path = _index_dir / _index_file_name; - InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_path.native()); + auto index_file_key = _inverted_index_file_reader->get_index_file_path(&_index_meta); + InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_key); if (InvertedIndexSearcherCache::instance()->lookup(searcher_cache_key, inverted_index_cache_handle)) { return Status::OK(); @@ -209,8 +206,20 @@ Status InvertedIndexReader::handle_searcher_cache( auto mem_tracker = std::make_unique("InvertedIndexSearcherCacheWithRead"); SCOPED_RAW_TIMER(&stats->inverted_index_searcher_open_timer); IndexSearcherPtr searcher; - RETURN_IF_ERROR(create_index_searcher(&searcher, _fs, _index_dir, _index_file_name, - mem_tracker.get(), type())); + bool exists = false; + RETURN_IF_ERROR(_inverted_index_file_reader->index_file_exist(&_index_meta, &exists)); + if (!exists) { + LOG(WARNING) << "inverted index: " << index_file_key << " not exist."; + return Status::Error( + "inverted index input file {} not found", index_file_key); + } + auto dir = DORIS_TRY(_inverted_index_file_reader->open(&_index_meta)); + // try to reuse index_searcher's directory to read null_bitmap to cache + // to avoid open directory additionally for null_bitmap + // TODO: handle null bitmap procedure in new format. + InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + static_cast(read_null_bitmap(&null_bitmap_cache_handle, dir.get())); + RETURN_IF_ERROR(create_index_searcher(dir.release(), &searcher, mem_tracker.get(), type())); auto* cache_value = new InvertedIndexSearcherCache::CacheValue( std::move(searcher), mem_tracker->consumption(), UnixMillis()); InvertedIndexSearcherCache::instance()->insert(searcher_cache_key, cache_value, @@ -219,30 +228,19 @@ Status InvertedIndexReader::handle_searcher_cache( } } -Status InvertedIndexReader::create_index_searcher(IndexSearcherPtr* searcher, io::FileSystemSPtr fs, - const io::Path& index_dir, - const std::string& index_file_name, +Status InvertedIndexReader::create_index_searcher(lucene::store::Directory* dir, + IndexSearcherPtr* searcher, MemTracker* mem_tracker, InvertedIndexReaderType reader_type) { - auto index_file_path = index_dir / index_file_name; - bool exists = false; - RETURN_IF_ERROR(fs->exists(index_file_path, &exists)); - if (!exists) { - LOG(WARNING) << "inverted index: " << index_file_path << " not exist."; - return Status::Error( - "inverted index input file {} not found", index_file_path.native()); - } SCOPED_CONSUME_MEM_TRACKER(mem_tracker); - bool open_idx_file_cache = true; - auto* directory = new DorisCompoundReader( - DorisCompoundDirectoryFactory::getDirectory(fs, index_dir.c_str()), - index_file_name.c_str(), config::inverted_index_read_buffer_size, open_idx_file_cache); - auto index_searcher_builder = DORIS_TRY(IndexSearcherBuilder::create_index_searcher_builder(reader_type)); - auto searcher_result = DORIS_TRY(index_searcher_builder->get_index_searcher(directory)); + auto searcher_result = DORIS_TRY(index_searcher_builder->get_index_searcher(dir)); *searcher = searcher_result; + if (std::string(dir->getObjectName()) == "DorisCompoundReader") { + static_cast(dir)->getDorisIndexInput()->setIdxFileCache(false); + } return Status::OK(); }; @@ -264,9 +262,10 @@ Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState* run try { std::vector analyse_result; InvertedIndexQueryCache::CacheKey cache_key; + auto index_file_path = _inverted_index_file_reader->get_index_file_path(&_index_meta); if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) { - cache_key = {_index_dir / _index_file_name, column_name, query_type, search_str}; + cache_key = {index_file_path, column_name, query_type, search_str}; analyse_result.emplace_back(search_str); } else { InvertedIndexCtxSPtr inverted_index_ctx = std::make_shared( @@ -311,7 +310,7 @@ Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState* run query_type == InvertedIndexQueryType::MATCH_ANY_QUERY) { std::string str_tokens = join(analyse_result, " "); - cache_key = {_index_dir / _index_file_name, column_name, query_type, str_tokens}; + cache_key = {index_file_path, column_name, query_type, str_tokens}; } auto* cache = InvertedIndexQueryCache::instance(); InvertedIndexQueryCacheHandle cache_handler; @@ -403,7 +402,7 @@ Status StringTypeInvertedIndexReader::query(OlapReaderStatistics* stats, [](lucene::index::Term* term) { _CLDECDELETE(term); }}; std::unique_ptr query; - auto index_file_path = _index_dir / _index_file_name; + auto index_file_path = _inverted_index_file_reader->get_index_file_path(&_index_meta); // try to get query bitmap result from cache and return immediately on cache hit InvertedIndexQueryCache::CacheKey cache_key {index_file_path, column_name, query_type, @@ -423,12 +422,6 @@ Status StringTypeInvertedIndexReader::query(OlapReaderStatistics* stats, auto searcher_variant = inverted_index_cache_handle.get_index_searcher(); searcher_ptr = std::get_if(&searcher_variant); if (searcher_ptr != nullptr) { - // try to reuse index_searcher's directory to read null_bitmap to cache - // to avoid open directory additionally for null_bitmap - InvertedIndexQueryCacheHandle null_bitmap_cache_handle; - RETURN_IF_ERROR(read_null_bitmap(&null_bitmap_cache_handle, - (*searcher_ptr)->getReader()->directory())); - try { switch (query_type) { case InvertedIndexQueryType::MATCH_ANY_QUERY: @@ -648,18 +641,19 @@ Status BkdIndexReader::try_query(OlapReaderStatistics* stats, const std::string& const void* query_value, InvertedIndexQueryType query_type, uint32_t* count) { try { + auto index_file_path = _inverted_index_file_reader->get_index_file_path(&_index_meta); + std::shared_ptr r; auto st = get_bkd_reader(r, stats); if (!st.ok()) { - LOG(WARNING) << "get bkd reader for " << _index_dir / _index_file_name - << " failed: " << st; + LOG(WARNING) << "get bkd reader for " << index_file_path << " failed: " << st; return st; } std::string query_str; _value_key_coder->full_encode_ascending(query_value, &query_str); - InvertedIndexQueryCache::CacheKey cache_key {_index_dir / _index_file_name, column_name, - query_type, query_str}; + InvertedIndexQueryCache::CacheKey cache_key {index_file_path, column_name, query_type, + query_str}; auto* cache = InvertedIndexQueryCache::instance(); InvertedIndexQueryCacheHandle cache_handler; roaring::Roaring bit_map; @@ -685,18 +679,19 @@ Status BkdIndexReader::query(OlapReaderStatistics* stats, RuntimeState* runtime_ SCOPED_RAW_TIMER(&stats->inverted_index_query_timer); try { + auto index_file_path = _inverted_index_file_reader->get_index_file_path(&_index_meta); + std::shared_ptr r; auto st = get_bkd_reader(r, stats); if (!st.ok()) { - LOG(WARNING) << "get bkd reader for " << _index_dir / _index_file_name - << " failed: " << st; + LOG(WARNING) << "get bkd reader for " << index_file_path << " failed: " << st; return st; } std::string query_str; _value_key_coder->full_encode_ascending(query_value, &query_str); - InvertedIndexQueryCache::CacheKey cache_key {_index_dir / _index_file_name, column_name, - query_type, query_str}; + InvertedIndexQueryCache::CacheKey cache_key {index_file_path, column_name, query_type, + query_str}; auto* cache = InvertedIndexQueryCache::instance(); InvertedIndexQueryCacheHandle cache_handler; auto cache_status = handle_query_cache(cache, cache_key, &cache_handler, stats, bit_map); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_reader.h index b8b40eece5..2db7e77b43 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include #include @@ -71,17 +70,15 @@ namespace segment_v2 { class InvertedIndexIterator; class InvertedIndexQueryCacheHandle; +class InvertedIndexFileReader; class InvertedIndexReader : public std::enable_shared_from_this { public: - explicit InvertedIndexReader(io::FileSystemSPtr fs, const std::string& path, - const TabletIndex* index_meta) - : _fs(fs), _path(path), _index_meta(*index_meta) { - io::Path io_path(_path); - _index_dir = io_path.parent_path(); - _index_file_name = InvertedIndexDescriptor::get_index_file_name( - io_path.filename(), index_meta->index_id(), index_meta->get_index_suffix()); - } + explicit InvertedIndexReader( + const TabletIndex* index_meta, + std::shared_ptr inverted_index_file_reader) + : _inverted_index_file_reader(std::move(inverted_index_file_reader)), + _index_meta(*index_meta) {} virtual ~InvertedIndexReader() = default; // create a new column iterator. Client should delete returned iterator @@ -106,6 +103,7 @@ public: } [[nodiscard]] bool has_null() const { return _has_null; } + void set_has_null(bool has_null) { _has_null = has_null; } static void get_analyse_result(std::vector& analyse_result, lucene::util::Reader* reader, @@ -135,28 +133,25 @@ public: virtual Status handle_searcher_cache(InvertedIndexCacheHandle* inverted_index_cache_handle, OlapReaderStatistics* stats); - static Status create_index_searcher(IndexSearcherPtr* searcher, io::FileSystemSPtr fs, - const io::Path& index_dir, - const std::string& index_file_name, MemTracker* mem_tracker, + static Status create_index_searcher(lucene::store::Directory* dir, IndexSearcherPtr* searcher, + MemTracker* mem_tracker, InvertedIndexReaderType reader_type); protected: friend class InvertedIndexIterator; - io::FileSystemSPtr _fs; - const std::string& _path; + std::shared_ptr _inverted_index_file_reader; TabletIndex _index_meta; - std::string _index_file_name; - io::Path _index_dir; - bool _has_null {true}; + bool _has_null = true; }; class FullTextIndexReader : public InvertedIndexReader { ENABLE_FACTORY_CREATOR(FullTextIndexReader); public: - explicit FullTextIndexReader(io::FileSystemSPtr fs, const std::string& path, - const TabletIndex* index_meta) - : InvertedIndexReader(fs, path, index_meta) {} + explicit FullTextIndexReader( + const TabletIndex* index_meta, + std::shared_ptr& inverted_index_file_reader) + : InvertedIndexReader(index_meta, inverted_index_file_reader) {} ~FullTextIndexReader() override = default; Status new_iterator(OlapReaderStatistics* stats, RuntimeState* runtime_state, @@ -179,18 +174,16 @@ private: const std::vector& analyse_result, const FulltextIndexSearcherPtr& index_searcher, const std::shared_ptr& term_match_bitmap); - - void check_null_bitmap(const FulltextIndexSearcherPtr& index_searcher, - bool& null_bitmap_already_read); }; class StringTypeInvertedIndexReader : public InvertedIndexReader { ENABLE_FACTORY_CREATOR(StringTypeInvertedIndexReader); public: - explicit StringTypeInvertedIndexReader(io::FileSystemSPtr fs, const std::string& path, - const TabletIndex* index_meta) - : InvertedIndexReader(fs, path, index_meta) {} + explicit StringTypeInvertedIndexReader( + const TabletIndex* index_meta, + std::shared_ptr& inverted_index_file_reader) + : InvertedIndexReader(index_meta, inverted_index_file_reader) {} ~StringTypeInvertedIndexReader() override = default; Status new_iterator(OlapReaderStatistics* stats, RuntimeState* runtime_state, @@ -246,9 +239,9 @@ class BkdIndexReader : public InvertedIndexReader { ENABLE_FACTORY_CREATOR(BkdIndexReader); public: - explicit BkdIndexReader(io::FileSystemSPtr fs, const std::string& path, - const TabletIndex* index_meta) - : InvertedIndexReader(fs, path, index_meta) {} + explicit BkdIndexReader(const TabletIndex* index_meta, + std::shared_ptr& inverted_index_file_reader) + : InvertedIndexReader(index_meta, inverted_index_file_reader) {} ~BkdIndexReader() override = default; Status new_iterator(OlapReaderStatistics* stats, RuntimeState* runtime_state, @@ -284,7 +277,7 @@ class InvertedIndexIterator { public: InvertedIndexIterator(OlapReaderStatistics* stats, RuntimeState* runtime_state, std::shared_ptr reader) - : _stats(stats), _runtime_state(runtime_state), _reader(reader) {} + : _stats(stats), _runtime_state(runtime_state), _reader(std::move(reader)) {} Status read_from_inverted_index(const std::string& column_name, const void* query_value, InvertedIndexQueryType query_type, uint32_t segment_num_rows, diff --git a/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp b/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp index 491d3e518e..aff9c6ae0e 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp @@ -21,12 +21,12 @@ #include #include "common/config.h" -#include "olap/rowset/segment_v2/inverted_index_compound_directory.h" #include "olap/rowset/segment_v2/inverted_index_compound_reader.h" #include "olap/rowset/segment_v2/inverted_index_desc.h" +#include "olap/rowset/segment_v2/inverted_index_fs_directory.h" namespace doris::segment_v2 { -Status FulltextIndexSearcherBuilder::build(DorisCompoundReader* directory, +Status FulltextIndexSearcherBuilder::build(lucene::store::Directory* directory, OptionalIndexSearcherPtr& output_searcher) { auto close_directory = true; lucene::index::IndexReader* reader = nullptr; @@ -51,15 +51,14 @@ Status FulltextIndexSearcherBuilder::build(DorisCompoundReader* directory, return Status::OK(); } -Status BKDIndexSearcherBuilder::build(DorisCompoundReader* directory, +Status BKDIndexSearcherBuilder::build(lucene::store::Directory* directory, OptionalIndexSearcherPtr& output_searcher) { try { auto close_directory = true; auto bkd_reader = std::make_shared(directory, close_directory); if (!bkd_reader->open()) { - LOG(INFO) << "bkd index file " << directory->getPath() + "/" + directory->getFileName() - << " is empty"; + LOG(INFO) << "bkd index file " << directory->toString() << " is empty"; } output_searcher = bkd_reader; _CLDECDELETE(directory) @@ -94,7 +93,8 @@ Result> IndexSearcherBuilder::create_index return index_builder; } -Result IndexSearcherBuilder::get_index_searcher(DorisCompoundReader* directory) { +Result IndexSearcherBuilder::get_index_searcher( + lucene::store::Directory* directory) { OptionalIndexSearcherPtr result; auto st = build(directory, result); if (!st.ok()) { @@ -106,7 +106,6 @@ Result IndexSearcherBuilder::get_index_searcher(DorisCompoundR return ResultError(Status::Error( "InvertedIndexSearcherCache build error.")); } - directory->getDorisIndexInput()->setIdxFileCache(false); return *result; } } // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/inverted_index_searcher.h b/be/src/olap/rowset/segment_v2/inverted_index_searcher.h index ac32bbda7d..b0466ab2df 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_searcher.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_searcher.h @@ -17,7 +17,14 @@ #pragma once -#include +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wshadow-field" +#endif +#include // IWYU pragma: keep +#ifdef __clang__ +#pragma clang diagnostic pop +#endif #include #include @@ -48,23 +55,23 @@ class DorisCompoundReader; class IndexSearcherBuilder { public: - virtual Status build(DorisCompoundReader* directory, + virtual Status build(lucene::store::Directory* directory, OptionalIndexSearcherPtr& output_searcher) = 0; virtual ~IndexSearcherBuilder() = default; - virtual Result get_index_searcher(DorisCompoundReader* directory); + virtual Result get_index_searcher(lucene::store::Directory* directory); static Result> create_index_searcher_builder( InvertedIndexReaderType reader_type); }; class FulltextIndexSearcherBuilder : public IndexSearcherBuilder { public: - Status build(DorisCompoundReader* directory, + Status build(lucene::store::Directory* directory, OptionalIndexSearcherPtr& output_searcher) override; }; class BKDIndexSearcherBuilder : public IndexSearcherBuilder { public: - Status build(DorisCompoundReader* directory, + Status build(lucene::store::Directory* directory, OptionalIndexSearcherPtr& output_searcher) override; }; } // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp index 9cd9897f6f..0c628f0651 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp @@ -39,6 +39,8 @@ #pragma clang diagnostic pop #endif +#include "common/config.h" +#include "gutil/strings/strip.h" #include "olap/field.h" #include "olap/inverted_index_parser.h" #include "olap/key_coder.h" @@ -46,8 +48,9 @@ #include "olap/rowset/segment_v2/common.h" #include "olap/rowset/segment_v2/inverted_index/char_filter/char_filter_factory.h" #include "olap/rowset/segment_v2/inverted_index_cache.h" -#include "olap/rowset/segment_v2/inverted_index_compound_directory.h" #include "olap/rowset/segment_v2/inverted_index_desc.h" +#include "olap/rowset/segment_v2/inverted_index_file_writer.h" +#include "olap/rowset/segment_v2/inverted_index_fs_directory.h" #include "olap/rowset/segment_v2/inverted_index_reader.h" #include "olap/tablet_schema.h" #include "olap/types.h" @@ -76,13 +79,9 @@ public: using CppType = typename CppTypeTraits::CppType; explicit InvertedIndexColumnWriterImpl(const std::string& field_name, - const std::string& segment_file_name, - const std::string& dir, const io::FileSystemSPtr& fs, + InvertedIndexFileWriter* index_file_writer, const TabletIndex* index_meta) - : _segment_file_name(segment_file_name), - _directory(dir), - _fs(fs), - _index_meta(index_meta) { + : _index_meta(index_meta), _index_file_writer(index_file_writer) { _parser_type = get_inverted_index_parser_type_from_string( get_parser_string_from_properties(_index_meta->properties())); _value_key_coder = get_key_coder(field_type); @@ -114,22 +113,21 @@ public: // open index searcher into cache auto mem_tracker = std::make_unique("InvertedIndexSearcherCacheWithRead"); - io::Path index_dir(_directory); - auto index_file_name = InvertedIndexDescriptor::get_index_file_name( - _segment_file_name, _index_meta->index_id(), - _index_meta->get_index_suffix()); + auto index_file_path = _index_file_writer->get_index_file_path(_index_meta); + auto index_tmp_dir = _dir->getDirName(); + InvertedIndexSearcherCache::CacheKey searcher_cache_key(index_file_path); + + auto* dir = DorisFSDirectoryFactory::getDirectory(_index_file_writer->get_fs(), + index_tmp_dir.c_str()); IndexSearcherPtr searcher; auto st = InvertedIndexReader::create_index_searcher( - &searcher, _fs, index_dir, index_file_name, mem_tracker.get(), - InvertedIndexReaderType::FULLTEXT); + dir, &searcher, mem_tracker.get(), InvertedIndexReaderType::FULLTEXT); if (UNLIKELY(!st.ok())) { LOG(ERROR) << "insert inverted index searcher cache error:" << st; return; } auto* cache_value = new InvertedIndexSearcherCache::CacheValue( std::move(searcher), mem_tracker->consumption(), UnixMillis()); - InvertedIndexSearcherCache::CacheKey searcher_cache_key( - (index_dir / index_file_name).native()); InvertedIndexSearcherCache::instance()->insert(searcher_cache_key, cache_value); } } @@ -142,11 +140,6 @@ public: } if (_dir) { _dir->deleteDirectory(); - io::Path cfs_path(_dir->getCfsDirName()); - auto idx_path = cfs_path.parent_path(); - std::string idx_name = std::string(cfs_path.stem().c_str()) + - DorisCompoundDirectory::COMPOUND_FILE_EXTENSION; - _dir->deleteFile(idx_name.c_str()); } } catch (CLuceneError& e) { LOG(ERROR) << "InvertedIndexWriter close_on_error failure: " << e.what(); @@ -161,7 +154,7 @@ public: _bkd_writer = std::make_shared( max_doc, DIMS, DIMS, value_length, MAX_LEAF_COUNT, MAXMBSortInHeap, total_point_count, true, config::max_depth_in_bkd_tree); - return create_index_directory(_dir); + return open_index_directory(); } std::unique_ptr create_chinese_analyzer() { @@ -194,32 +187,8 @@ public: return Status::OK(); } - Status create_index_directory(std::unique_ptr& dir) { - bool use_compound_file_writer = true; - bool can_use_ram_dir = true; - auto index_path = InvertedIndexDescriptor::get_temporary_index_path( - _directory + "/" + _segment_file_name, _index_meta->index_id(), - _index_meta->get_index_suffix()); - - bool exists = false; - auto st = _fs->exists(index_path.c_str(), &exists); - if (!st.ok()) { - LOG(ERROR) << "index_path: exists error:" << st; - return st; - } - if (exists) { - LOG(ERROR) << "try to init a directory:" << index_path << " already exists"; - return Status::InternalError("init_fulltext_index directory already exists"); - } - - auto tmp_file_dir = ExecEnv::GetInstance()->get_tmp_file_dirs()->get_tmp_file_dir(); - _lfs = io::global_local_filesystem(); - auto lfs_index_path = InvertedIndexDescriptor::get_temporary_index_path( - tmp_file_dir / _segment_file_name, _index_meta->index_id(), - _index_meta->get_index_suffix()); - dir = std::unique_ptr(DorisCompoundDirectoryFactory::getDirectory( - _lfs, lfs_index_path.c_str(), use_compound_file_writer, can_use_ram_dir, nullptr, - _fs, index_path.c_str())); + Status open_index_directory() { + _dir = DORIS_TRY(_index_file_writer->open(_index_meta)); return Status::OK(); } @@ -227,9 +196,9 @@ public: bool create_index = true; bool close_dir_on_shutdown = true; index_writer = std::make_unique( - _dir.get(), _analyzer.get(), create_index, close_dir_on_shutdown); + _dir, _analyzer.get(), create_index, close_dir_on_shutdown); index_writer->setRAMBufferSizeMB(config::inverted_index_ram_buffer_size); - _index_writer->setMaxBufferedDocs(config::inverted_index_max_buffered_docs); + index_writer->setMaxBufferedDocs(config::inverted_index_max_buffered_docs); index_writer->setMaxFieldLength(MAX_FIELD_LEN); index_writer->setMergeFactor(MERGE_FACTOR); index_writer->setUseCompoundFile(false); @@ -245,10 +214,8 @@ public: : int(lucene::document::Field::INDEX_TOKENIZED); *field = new lucene::document::Field(_field_name.c_str(), field_config); (*field)->setOmitTermFreqAndPositions( - get_parser_phrase_support_string_from_properties(_index_meta->properties()) == - INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES - ? false - : true); + !(get_parser_phrase_support_string_from_properties(_index_meta->properties()) == + INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES)); return Status::OK(); } @@ -287,7 +254,7 @@ public: } Status init_fulltext_index() { - RETURN_IF_ERROR(create_index_directory(_dir)); + RETURN_IF_ERROR(open_index_directory()); RETURN_IF_ERROR(create_char_string_reader(_char_string_reader)); RETURN_IF_ERROR(create_analyzer(_analyzer)); RETURN_IF_ERROR(create_index_writer(_index_writer)); @@ -302,7 +269,6 @@ public: _index_writer->addDocument(_doc.get()); } catch (const CLuceneError& e) { close_on_error(); - _dir->deleteDirectory(); return Status::Error( "CLuceneError add_document: {}", e.what()); } @@ -314,7 +280,6 @@ public: _index_writer->addNullDocument(_doc.get()); } catch (const CLuceneError& e) { close_on_error(); - _dir->deleteDirectory(); return Status::Error( "CLuceneError add_null_document: {}", e.what()); } @@ -351,12 +316,12 @@ public: void new_char_token_stream(const char* s, size_t len, lucene::document::Field* field) { _char_string_reader->init(s, len, false); - auto stream = _analyzer->reusableTokenStream(field->name(), _char_string_reader.get()); + auto* stream = _analyzer->reusableTokenStream(field->name(), _char_string_reader.get()); field->setValue(stream); } void new_field_value(const char* s, size_t len, lucene::document::Field* field) { - auto field_value = lucene::util::Misc::_charToWide(s, len); + auto* field_value = lucene::util::Misc::_charToWide(s, len); field->setValue(field_value, false); // setValue did not duplicate value, so we don't have to delete //_CLDELETE_ARRAY(field_value) @@ -546,72 +511,85 @@ public: faststring buf; buf.resize(size); _null_bitmap.write(reinterpret_cast(buf.data()), false); - null_bitmap_out->writeBytes(reinterpret_cast(buf.data()), size); + null_bitmap_out->writeBytes(buf.data(), size); null_bitmap_out->close(); } } Status finish() override { - std::unique_ptr null_bitmap_out = nullptr; - std::unique_ptr data_out = nullptr; - std::unique_ptr index_out = nullptr; - std::unique_ptr meta_out = nullptr; - try { - // write bkd file - if constexpr (field_is_numeric_type(field_type)) { - _bkd_writer->max_doc_ = _rid; - _bkd_writer->docs_seen_ = _row_ids_seen_for_bkd; - null_bitmap_out = std::unique_ptr(_dir->createOutput( - InvertedIndexDescriptor::get_temporary_null_bitmap_file_name().c_str())); - data_out = std::unique_ptr(_dir->createOutput( - InvertedIndexDescriptor::get_temporary_bkd_index_data_file_name().c_str())); - meta_out = std::unique_ptr(_dir->createOutput( - InvertedIndexDescriptor::get_temporary_bkd_index_meta_file_name().c_str())); - index_out = std::unique_ptr(_dir->createOutput( - InvertedIndexDescriptor::get_temporary_bkd_index_file_name().c_str())); - write_null_bitmap(null_bitmap_out.get()); + if (_dir != nullptr) { + std::unique_ptr null_bitmap_out = nullptr; + std::unique_ptr data_out = nullptr; + std::unique_ptr index_out = nullptr; + std::unique_ptr meta_out = nullptr; + try { + // write bkd file + if constexpr (field_is_numeric_type(field_type)) { + _bkd_writer->max_doc_ = _rid; + _bkd_writer->docs_seen_ = _row_ids_seen_for_bkd; + null_bitmap_out = + std::unique_ptr(_dir->createOutput( + InvertedIndexDescriptor::get_temporary_null_bitmap_file_name() + .c_str())); + data_out = std::unique_ptr(_dir->createOutput( + InvertedIndexDescriptor::get_temporary_bkd_index_data_file_name() + .c_str())); + meta_out = std::unique_ptr(_dir->createOutput( + InvertedIndexDescriptor::get_temporary_bkd_index_meta_file_name() + .c_str())); + index_out = std::unique_ptr(_dir->createOutput( + InvertedIndexDescriptor::get_temporary_bkd_index_file_name().c_str())); + write_null_bitmap(null_bitmap_out.get()); - DBUG_EXECUTE_IF("InvertedIndexWriter._set_bkd_data_out_nullptr", - { data_out = nullptr; }); - if (data_out != nullptr && meta_out != nullptr && index_out != nullptr) { - _bkd_writer->meta_finish(meta_out.get(), - _bkd_writer->finish(data_out.get(), index_out.get()), - int(field_type)); - } else { - LOG(WARNING) << "Inverted index writer create output error occurred: nullptr"; - _CLTHROWA(CL_ERR_IO, "Create output error with nullptr"); + DBUG_EXECUTE_IF("InvertedIndexWriter._set_bkd_data_out_nullptr", + { data_out = nullptr; }); + if (data_out != nullptr && meta_out != nullptr && index_out != nullptr) { + _bkd_writer->meta_finish( + meta_out.get(), + _bkd_writer->finish(data_out.get(), index_out.get()), + int(field_type)); + } else { + LOG(WARNING) + << "Inverted index writer create output error occurred: nullptr"; + _CLTHROWA(CL_ERR_IO, "Create output error with nullptr"); + } + meta_out->close(); + data_out->close(); + index_out->close(); + _dir->close(); + } else if constexpr (field_is_slice_type(field_type)) { + null_bitmap_out = + std::unique_ptr(_dir->createOutput( + InvertedIndexDescriptor::get_temporary_null_bitmap_file_name() + .c_str())); + write_null_bitmap(null_bitmap_out.get()); + close(); + DBUG_EXECUTE_IF( + "InvertedIndexWriter._throw_clucene_error_in_fulltext_writer_close", { + _CLTHROWA(CL_ERR_IO, + "debug point: test throw error in fulltext index writer"); + }); } - meta_out->close(); - data_out->close(); - index_out->close(); - _dir->close(); - } else if constexpr (field_is_slice_type(field_type)) { - null_bitmap_out = std::unique_ptr(_dir->createOutput( - InvertedIndexDescriptor::get_temporary_null_bitmap_file_name().c_str())); - write_null_bitmap(null_bitmap_out.get()); - close(); - DBUG_EXECUTE_IF( - "InvertedIndexWriter._throw_clucene_error_in_fulltext_writer_close", { - _CLTHROWA(CL_ERR_IO, - "debug point: test throw error in fulltext index writer"); - }); + } catch (CLuceneError& e) { + FINALLY_CLOSE_OUTPUT(null_bitmap_out) + FINALLY_CLOSE_OUTPUT(meta_out) + FINALLY_CLOSE_OUTPUT(data_out) + FINALLY_CLOSE_OUTPUT(index_out) + if constexpr (field_is_numeric_type(field_type)) { + FINALLY_CLOSE_OUTPUT(_dir) + } else if constexpr (field_is_slice_type(field_type)) { + FINALLY_CLOSE_OUTPUT(_index_writer); + } + LOG(WARNING) << "Inverted index writer finish error occurred: " << e.what(); + return Status::Error( + "Inverted index writer finish error occurred:{}", e.what()); } - } catch (CLuceneError& e) { - FINALLY_CLOSE_OUTPUT(null_bitmap_out) - FINALLY_CLOSE_OUTPUT(meta_out) - FINALLY_CLOSE_OUTPUT(data_out) - FINALLY_CLOSE_OUTPUT(index_out) - if constexpr (field_is_numeric_type(field_type)) { - FINALLY_CLOSE_OUTPUT(_dir) - } else if constexpr (field_is_slice_type(field_type)) { - FINALLY_CLOSE_OUTPUT(_index_writer); - } - LOG(WARNING) << "Inverted index writer finish error occurred: " << e.what(); - return Status::Error( - "Inverted index writer finish error occurred"); - } - return Status::OK(); + return Status::OK(); + } + LOG(WARNING) << "Inverted index writer finish error occurred: dir is nullptr"; + return Status::Error( + "Inverted index writer finish error occurred: dir is nullptr"); } private: @@ -626,22 +604,18 @@ private: std::unique_ptr _analyzer = nullptr; std::unique_ptr _char_string_reader = nullptr; std::shared_ptr _bkd_writer = nullptr; - std::unique_ptr _dir = nullptr; - std::string _segment_file_name; - std::string _directory; - io::FileSystemSPtr _fs; - io::FileSystemSPtr _lfs; + DorisFSDirectory* _dir = nullptr; const KeyCoder* _value_key_coder; const TabletIndex* _index_meta; InvertedIndexParserType _parser_type; std::wstring _field_name; + InvertedIndexFileWriter* _index_file_writer; }; Status InvertedIndexColumnWriter::create(const Field* field, std::unique_ptr* res, - const std::string& segment_file_name, - const std::string& dir, const TabletIndex* index_meta, - const io::FileSystemSPtr& fs) { + InvertedIndexFileWriter* index_file_writer, + const TabletIndex* index_meta) { const auto* typeinfo = field->type_info(); FieldType type = typeinfo->type(); std::string field_name = field->name(); @@ -660,7 +634,7 @@ Status InvertedIndexColumnWriter::create(const Field* field, #define M(TYPE) \ case TYPE: \ *res = std::make_unique>( \ - field_name, segment_file_name, dir, fs, index_meta); \ + field_name, index_file_writer, index_meta); \ break; M(FieldType::OLAP_FIELD_TYPE_TINYINT) M(FieldType::OLAP_FIELD_TYPE_SMALLINT) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.h b/be/src/olap/rowset/segment_v2/inverted_index_writer.h index fee81f8235..5ed34852c9 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.h @@ -40,12 +40,13 @@ class Field; class TabletIndex; namespace segment_v2 { +class InvertedIndexFileWriter; class InvertedIndexColumnWriter { public: static Status create(const Field* field, std::unique_ptr* res, - const std::string& segment_file_name, const std::string& dir, - const TabletIndex* inverted_index, const io::FileSystemSPtr& fs); + InvertedIndexFileWriter* index_file_writer, + const TabletIndex* inverted_index); virtual Status init() = 0; InvertedIndexColumnWriter() = default; diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index 41f1d0b7d2..e74c5d2a6b 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -41,6 +41,7 @@ #include "olap/rowset/segment_v2/empty_segment_iterator.h" #include "olap/rowset/segment_v2/hierarchical_data_reader.h" #include "olap/rowset/segment_v2/indexed_column_reader.h" +#include "olap/rowset/segment_v2/inverted_index_file_reader.h" #include "olap/rowset/segment_v2/page_io.h" #include "olap/rowset/segment_v2/page_pointer.h" #include "olap/rowset/segment_v2/segment_iterator.h" @@ -115,6 +116,21 @@ Status Segment::_open() { return Status::OK(); } +Status Segment::_open_inverted_index() { + _inverted_index_file_reader = std::make_shared( + _file_reader->fs(), _file_reader->path().parent_path(), + _file_reader->path().filename().native(), + _tablet_schema->get_inverted_index_storage_format()); + bool open_idx_file_cache = true; + auto st = _inverted_index_file_reader->init(config::inverted_index_read_buffer_size, + open_idx_file_cache); + if (st.is()) { + LOG(INFO) << st; + return Status::OK(); + } + return st; +} + Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_options, std::unique_ptr* iter) { read_options.stats->total_segment_number++; @@ -180,7 +196,7 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o read_options.push_down_agg_type_opt != TPushAggOp::COUNT_ON_INDEX) { iter->reset(vectorized::new_vstatistics_iterator(this->shared_from_this(), *schema)); } else { - iter->reset(new SegmentIterator(this->shared_from_this(), schema)); + *iter = std::make_unique(this->shared_from_this(), schema); } if (config::ignore_always_true_predicate_for_segment && @@ -632,7 +648,12 @@ Status Segment::new_inverted_index_iterator(const TabletColumn& tablet_column, std::unique_ptr* iter) { ColumnReader* reader = _get_column_reader(tablet_column); if (reader != nullptr && index_meta) { - RETURN_IF_ERROR(reader->new_inverted_index_iterator(index_meta, read_options, iter)); + if (_inverted_index_file_reader == nullptr) { + RETURN_IF_ERROR( + _inverted_index_file_reader_open.call([&] { return _open_inverted_index(); })); + } + RETURN_IF_ERROR(reader->new_inverted_index_iterator(_inverted_index_file_reader, index_meta, + read_options, iter)); return Status::OK(); } return Status::OK(); diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h index af5d1896f4..f9d426c8de 100644 --- a/be/src/olap/rowset/segment_v2/segment.h +++ b/be/src/olap/rowset/segment_v2/segment.h @@ -66,6 +66,7 @@ namespace segment_v2 { class BitmapIndexIterator; class Segment; class InvertedIndexIterator; +class InvertedIndexFileReader; using SegmentSharedPtr = std::shared_ptr; // A Segment is used to represent a segment in memory format. When segment is @@ -201,6 +202,7 @@ private: vectorized::DataTypePtr target_type_hint); Status _load_index_impl(); + Status _open_inverted_index(); private: friend class SegmentIterator; @@ -245,6 +247,9 @@ private: // Segment may be destructed after StorageEngine, in order to exit gracefully. std::shared_ptr _segment_meta_mem_tracker; std::mutex _open_lock; + // inverted index file reader + std::shared_ptr _inverted_index_file_reader; + DorisCallOnce _inverted_index_file_reader_open; }; } // namespace segment_v2 diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 8023f3f2d7..d923b5e9d6 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -38,6 +38,7 @@ #include "common/logging.h" #include "common/object_pool.h" #include "common/status.h" +#include "io/fs/file_reader.h" #include "io/io_common.h" #include "olap/bloom_filter_predicate.h" #include "olap/column_predicate.h" @@ -50,6 +51,7 @@ #include "olap/rowset/segment_v2/bitmap_index_reader.h" #include "olap/rowset/segment_v2/column_reader.h" #include "olap/rowset/segment_v2/indexed_column_reader.h" +#include "olap/rowset/segment_v2/inverted_index_file_reader.h" #include "olap/rowset/segment_v2/inverted_index_reader.h" #include "olap/rowset/segment_v2/row_ranges.h" #include "olap/rowset/segment_v2/segment.h" @@ -1227,12 +1229,12 @@ Status SegmentIterator::_init_return_column_iterators() { std::vector tmp_is_pred_column; tmp_is_pred_column.resize(_schema->columns().size(), false); for (auto predicate : _col_predicates) { - auto cid = predicate->column_id(); - tmp_is_pred_column[cid] = true; + auto p_cid = predicate->column_id(); + tmp_is_pred_column[p_cid] = true; } // handle delete_condition - for (auto cid : del_cond_id_set) { - tmp_is_pred_column[cid] = true; + for (auto d_cid : del_cond_id_set) { + tmp_is_pred_column[d_cid] = true; } if (_column_iterators[cid] == nullptr) { diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index ff32d09b18..990fed3f28 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -32,6 +32,7 @@ #include "common/logging.h" // LOG #include "common/status.h" #include "gutil/port.h" +#include "inverted_index_fs_directory.h" #include "io/fs/file_writer.h" #include "olap/data_dir.h" #include "olap/key_coder.h" @@ -40,6 +41,7 @@ #include "olap/row_cursor.h" // RowCursor // IWYU pragma: keep #include "olap/rowset/rowset_writer_context.h" // RowsetWriterContext #include "olap/rowset/segment_v2/column_writer.h" // ColumnWriter +#include "olap/rowset/segment_v2/inverted_index_file_writer.h" #include "olap/rowset/segment_v2/page_io.h" #include "olap/rowset/segment_v2/page_pointer.h" #include "olap/segment_loader.h" @@ -131,6 +133,12 @@ SegmentWriter::SegmentWriter(io::FileWriter* file_writer, uint32_t segment_id, } } } + if (_tablet_schema->has_inverted_index()) { + _inverted_index_file_writer = std::make_unique( + _file_writer->fs(), _file_writer->path().parent_path(), + _file_writer->path().filename(), + _tablet_schema->get_inverted_index_storage_format()); + } } SegmentWriter::~SegmentWriter() { @@ -213,7 +221,7 @@ Status SegmentWriter::init(const std::vector& col_ids, bool has_key) { skip_inverted_index = true; } // indexes for this column - opts.indexes = _tablet_schema->get_indexes_for_column(column); + opts.indexes = std::move(_tablet_schema->get_indexes_for_column(column)); if (column.is_variant_type() || (column.is_extracted_column() && column.is_jsonb_type()) || (column.is_extracted_column() && column.is_array_type())) { // variant and jsonb type skip write index @@ -222,9 +230,11 @@ Status SegmentWriter::init(const std::vector& col_ids, bool has_key) { opts.need_bloom_filter = false; opts.need_bitmap_index = false; } + opts.inverted_index_file_writer = _inverted_index_file_writer.get(); for (auto index : opts.indexes) { - if (!skip_inverted_index && index && index->index_type() == IndexType::INVERTED) { + if (!skip_inverted_index && index->index_type() == IndexType::INVERTED) { opts.inverted_index = index; + opts.need_inverted_index = true; // TODO support multiple inverted index break; } @@ -991,11 +1001,10 @@ uint64_t SegmentWriter::estimate_segment_size() { } size_t SegmentWriter::try_get_inverted_index_file_size() { - size_t total_size = 0; - for (auto& column_writer : _column_writers) { - total_size += column_writer->get_inverted_index_size(); + if (_inverted_index_file_writer != nullptr) { + return _inverted_index_file_writer->get_index_file_size(); } - return total_size; + return 0; } Status SegmentWriter::finalize_columns_data() { @@ -1048,7 +1057,6 @@ Status SegmentWriter::finalize_columns_index(uint64_t* index_size) { *index_size = _file_writer->bytes_appended() - index_start; } } - _inverted_index_file_size = try_get_inverted_index_file_size(); // reset all column writers and data_conveter clear(); @@ -1063,6 +1071,10 @@ Status SegmentWriter::finalize_footer(uint64_t* segment_file_size) { if (*segment_file_size == 0) { return Status::Corruption("Bad segment, file size = 0"); } + if (_inverted_index_file_writer != nullptr) { + RETURN_IF_ERROR(_inverted_index_file_writer->close()); + } + _inverted_index_file_size = try_get_inverted_index_file_size(); return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/segment_writer.h b/be/src/olap/rowset/segment_v2/segment_writer.h index d5d3a11ba7..1adb94aad2 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.h +++ b/be/src/olap/rowset/segment_v2/segment_writer.h @@ -64,6 +64,7 @@ class FileWriter; } // namespace io namespace segment_v2 { +class InvertedIndexFileWriter; extern const char* k_segment_magic; extern const uint32_t k_segment_magic_length; @@ -184,6 +185,7 @@ private: // Not owned. owned by RowsetWriter io::FileWriter* _file_writer = nullptr; + std::unique_ptr _inverted_index_file_writer; SegmentFooterPB _footer; size_t _num_key_columns; diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index b800edfb5e..f60c2eb259 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -30,6 +30,7 @@ #include "common/config.h" #include "common/logging.h" // LOG #include "gutil/port.h" +#include "inverted_index_fs_directory.h" #include "io/fs/file_writer.h" #include "olap/data_dir.h" #include "olap/key_coder.h" @@ -38,6 +39,7 @@ #include "olap/row_cursor.h" // RowCursor // IWYU pragma: keep #include "olap/rowset/rowset_writer_context.h" // RowsetWriterContext #include "olap/rowset/segment_v2/column_writer.h" // ColumnWriter +#include "olap/rowset/segment_v2/inverted_index_file_writer.h" #include "olap/rowset/segment_v2/page_io.h" #include "olap/rowset/segment_v2/page_pointer.h" #include "olap/segment_loader.h" @@ -103,6 +105,12 @@ VerticalSegmentWriter::VerticalSegmentWriter(io::FileWriter* file_writer, uint32 _tablet_schema->db_id(), _tablet_schema->table_id(), _tablet_schema->column(_tablet_schema->auto_increment_column()).unique_id()); } + if (_tablet_schema->has_inverted_index()) { + _inverted_index_file_writer = std::make_unique( + _file_writer->fs(), _file_writer->path().parent_path(), + _file_writer->path().filename(), + _tablet_schema->get_inverted_index_storage_format()); + } } VerticalSegmentWriter::~VerticalSegmentWriter() { @@ -174,12 +182,14 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo opts.need_bitmap_index = false; } for (auto index : opts.indexes) { - if (!skip_inverted_index && index && index->index_type() == IndexType::INVERTED) { + if (!skip_inverted_index && index->index_type() == IndexType::INVERTED) { opts.inverted_index = index; + opts.need_inverted_index = true; // TODO support multiple inverted index break; } } + opts.inverted_index_file_writer = _inverted_index_file_writer.get(); #define CHECK_FIELD_TYPE(TYPE, type_name) \ if (column.type() == FieldType::OLAP_FIELD_TYPE_##TYPE) { \ @@ -886,11 +896,10 @@ uint64_t VerticalSegmentWriter::_estimated_remaining_size() { } size_t VerticalSegmentWriter::_calculate_inverted_index_file_size() { - size_t total_size = 0; - for (auto& column_writer : _column_writers) { - total_size += column_writer->get_inverted_index_size(); + if (_inverted_index_file_writer != nullptr) { + return _inverted_index_file_writer->get_index_file_size(); } - return total_size; + return 0; } Status VerticalSegmentWriter::finalize_columns_index(uint64_t* index_size) { @@ -986,6 +995,9 @@ Status VerticalSegmentWriter::_write_inverted_index() { for (auto& column_writer : _column_writers) { RETURN_IF_ERROR(column_writer->write_inverted_index()); } + if (_inverted_index_file_writer != nullptr) { + RETURN_IF_ERROR(_inverted_index_file_writer->close()); + } return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.h b/be/src/olap/rowset/segment_v2/vertical_segment_writer.h index 806b307548..7bc6bf7c4f 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.h +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.h @@ -59,6 +59,7 @@ class FileWriter; } // namespace io namespace segment_v2 { +class InvertedIndexFileWriter; struct VerticalSegmentWriterOptions { uint32_t num_rows_per_block = 1024; @@ -155,6 +156,7 @@ private: // Not owned. owned by RowsetWriter io::FileWriter* _file_writer = nullptr; + std::unique_ptr _inverted_index_file_writer; SegmentFooterPB _footer; size_t _num_key_columns; diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 20f05ac816..07cc07bc87 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -70,7 +70,8 @@ TabletMetaSharedPtr TabletMeta::create( request.time_series_compaction_goal_size_mbytes, request.time_series_compaction_file_count_threshold, request.time_series_compaction_time_threshold_seconds, - request.time_series_compaction_empty_rowsets_threshold); + request.time_series_compaction_empty_rowsets_threshold, + request.inverted_index_storage_format); } TabletMeta::TabletMeta() @@ -89,7 +90,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id int64_t time_series_compaction_goal_size_mbytes, int64_t time_series_compaction_file_count_threshold, int64_t time_series_compaction_time_threshold_seconds, - int64_t time_series_compaction_empty_rowsets_threshold) + int64_t time_series_compaction_empty_rowsets_threshold, + TInvertedIndexStorageFormat::type inverted_index_storage_format) : _tablet_uid(0, 0), _schema(new TabletSchema), _delete_bitmap(new DeleteBitmap(tablet_id)) { @@ -165,6 +167,18 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id break; } + switch (inverted_index_storage_format) { + case TInvertedIndexStorageFormat::V1: + schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); + break; + case TInvertedIndexStorageFormat::V2: + schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + break; + default: + schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); + break; + } + switch (tablet_schema.sort_type) { case TSortType::type::ZORDER: schema->set_sort_type(SortType::ZORDER); diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 877c41bda7..4400be42af 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -110,7 +110,9 @@ public: int64_t time_series_compaction_goal_size_mbytes = 1024, int64_t time_series_compaction_file_count_threshold = 2000, int64_t time_series_compaction_time_threshold_seconds = 3600, - int64_t time_series_compaction_empty_rowsets_threshold = 5); + int64_t time_series_compaction_empty_rowsets_threshold = 5, + TInvertedIndexStorageFormat::type inverted_index_storage_format = + TInvertedIndexStorageFormat::V1); // If need add a filed in TableMeta, filed init copy in copy construct function TabletMeta(const TabletMeta& tablet_meta); TabletMeta(TabletMeta&& tablet_meta) = delete; diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index d41532cf89..b0410a2b34 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -951,6 +951,12 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac _sort_col_num = schema.sort_col_num(); _compression_type = schema.compression_type(); _schema_version = schema.schema_version(); + // Default to V1 inverted index storage format for backward compatibility if not specified in schema. + if (!schema.has_inverted_index_storage_format()) { + _inverted_index_storage_format = InvertedIndexStorageFormatPB::V1; + } else { + _inverted_index_storage_format = schema.inverted_index_storage_format(); + } } void TabletSchema::copy_from(const TabletSchema& tablet_schema) { @@ -1029,8 +1035,8 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version _num_columns++; } - for (auto& index : index->indexes) { - _indexes.emplace_back(*index); + for (auto& i : index->indexes) { + _indexes.emplace_back(*i); } if (has_bf_columns) { @@ -1138,6 +1144,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { tablet_schema_pb->set_schema_version(_schema_version); tablet_schema_pb->set_compression_type(_compression_type); tablet_schema_pb->set_version_col_idx(_version_col_idx); + tablet_schema_pb->set_inverted_index_storage_format(_inverted_index_storage_format); } size_t TabletSchema::row_size() const { diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 5c36122ffc..dae7c86051 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -333,6 +333,14 @@ public: segment_v2::CompressionTypePB compression_type() const { return _compression_type; } const std::vector& indexes() const { return _indexes; } + bool has_inverted_index() const { + for (const auto& index : _indexes) { + if (index.index_type() == IndexType::INVERTED) { + return true; + } + } + return false; + } std::vector get_indexes_for_column(const TabletColumn& col) const; bool has_inverted_index(const TabletColumn& col) const; bool has_inverted_index_with_index_id(int32_t index_id, const std::string& suffix_path) const; @@ -421,6 +429,9 @@ public: vectorized::Block create_block_by_cids(const std::vector& cids); std::shared_ptr copy_without_extracted_columns(); + InvertedIndexStorageFormatPB get_inverted_index_storage_format() const { + return _inverted_index_storage_format; + } private: friend bool operator==(const TabletSchema& a, const TabletSchema& b); @@ -462,6 +473,7 @@ private: int64_t _mem_size = 0; bool _store_row_column = false; bool _skip_write_index_on_load = false; + InvertedIndexStorageFormatPB _inverted_index_storage_format = InvertedIndexStorageFormatPB::V1; }; bool operator==(const TabletSchema& a, const TabletSchema& b); diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp index 5b6b73f95a..4fd785d029 100644 --- a/be/src/olap/task/index_builder.cpp +++ b/be/src/olap/task/index_builder.cpp @@ -19,13 +19,14 @@ #include "common/status.h" #include "olap/rowset/beta_rowset.h" -#include "olap/rowset/rowset_writer.h" #include "olap/rowset/rowset_writer_context.h" +#include "olap/rowset/segment_v2/inverted_index_file_reader.h" +#include "olap/rowset/segment_v2/inverted_index_file_writer.h" +#include "olap/rowset/segment_v2/inverted_index_fs_directory.h" +#include "olap/rowset/segment_v2/inverted_index_writer.h" #include "olap/segment_loader.h" #include "olap/storage_engine.h" #include "olap/tablet_schema.h" -#include "runtime/memory/mem_tracker.h" -#include "runtime/thread_context.h" #include "util/trace.h" namespace doris { @@ -64,39 +65,53 @@ Status IndexBuilder::update_inverted_index_info() { TabletSchemaSPtr output_rs_tablet_schema = std::make_shared(); const auto& input_rs_tablet_schema = input_rowset->tablet_schema(); output_rs_tablet_schema->copy_from(*input_rs_tablet_schema); + size_t total_index_size = 0; + auto* beta_rowset = reinterpret_cast(input_rowset.get()); + auto size_st = beta_rowset->get_inverted_index_size(&total_index_size); + if (!size_st.ok() && !size_st.is() && + !size_st.is()) { + return size_st; + } + auto num_segments = input_rowset->num_segments(); + size_t drop_index_size = 0; + if (_is_drop_op) { - size_t total_index_size = 0; for (const auto& t_inverted_index : _alter_inverted_indexes) { - auto* beta_rowset = reinterpret_cast(input_rowset.get()); - size_t index_size = 0; - RETURN_IF_ERROR(beta_rowset->get_inverted_index_size_by_index_id( - t_inverted_index.index_id, &index_size)); - total_index_size += index_size; + DCHECK_EQ(t_inverted_index.columns.size(), 1); + auto column_name = t_inverted_index.columns[0]; + auto column_idx = output_rs_tablet_schema->field_index(column_name); + if (column_idx < 0) { + LOG(WARNING) << "referenced column was missing. " + << "[column=" << column_name << " referenced_column=" << column_idx + << "]"; + continue; + } + auto column = output_rs_tablet_schema->column(column_idx); + const auto* index_meta = output_rs_tablet_schema->get_inverted_index(column); + if (index_meta == nullptr) { + LOG(ERROR) << "failed to find column: " << column_name + << " index_id: " << t_inverted_index.index_id; + continue; + } + if (output_rs_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::V1) { + for (int seg_id = 0; seg_id < num_segments; seg_id++) { + auto segment_full_path = + io::Path(static_cast(input_rowset.get()) + ->segment_file_path(seg_id)); + auto index_file_full_path = InvertedIndexDescriptor::get_index_file_name( + segment_full_path, index_meta->index_id(), + index_meta->get_index_suffix()); + int64_t index_size = 0; + RETURN_IF_ERROR(input_rowset->rowset_meta()->fs()->file_size( + index_file_full_path, &index_size)); + VLOG_DEBUG << "inverted index file:" << index_file_full_path + << " size:" << index_size; + drop_index_size += index_size; + } + } output_rs_tablet_schema->remove_index(t_inverted_index.index_id); } - - auto input_rowset_meta = input_rowset->rowset_meta(); - auto update_disk_size = [&](size_t& disk_size, const std::string& size_type) { - if (disk_size >= total_index_size) { - disk_size -= total_index_size; - } else { - LOG(WARNING) << "rowset " << input_rowset_meta->rowset_id() << " " << size_type - << " size:" << disk_size - << " is less than index size:" << total_index_size; - } - }; - - size_t before_size = input_rowset_meta->total_disk_size(); - update_disk_size(before_size, "total disk"); - input_rowset_meta->set_total_disk_size(before_size); - - before_size = input_rowset_meta->data_disk_size(); - update_disk_size(before_size, "data disk"); - input_rowset_meta->set_data_disk_size(before_size); - - before_size = input_rowset_meta->index_disk_size(); - update_disk_size(before_size, "index"); - input_rowset_meta->set_index_disk_size(before_size); } else { // base on input rowset's tablet_schema to build // output rowset's tablet_schema which only add @@ -122,7 +137,6 @@ Status IndexBuilder::update_inverted_index_info() { // construct input rowset reader RowsetReaderSharedPtr input_rs_reader; RETURN_IF_ERROR(input_rowset->create_reader(&input_rs_reader)); - // construct output rowset writer RowsetWriterContext context; context.version = input_rs_reader->version(); @@ -148,9 +162,47 @@ Status IndexBuilder::update_inverted_index_info() { auto input_rowset_meta = input_rowset->rowset_meta(); RowsetMetaSharedPtr rowset_meta = std::make_shared(); rowset_meta->set_num_rows(input_rowset_meta->num_rows()); - rowset_meta->set_total_disk_size(input_rowset_meta->total_disk_size()); - rowset_meta->set_data_disk_size(input_rowset_meta->data_disk_size()); - rowset_meta->set_index_disk_size(input_rowset_meta->index_disk_size()); + if (output_rs_tablet_schema->get_inverted_index_storage_format() == + InvertedIndexStorageFormatPB::V1) { + if (_is_drop_op) { + VLOG_DEBUG << "data_disk_size:" << input_rowset_meta->data_disk_size() + << " total_disk_size:" << input_rowset_meta->data_disk_size() + << " index_disk_size:" << input_rowset_meta->index_disk_size() + << " drop_index_size:" << drop_index_size; + rowset_meta->set_total_disk_size(input_rowset_meta->total_disk_size() - + drop_index_size); + rowset_meta->set_data_disk_size(input_rowset_meta->data_disk_size() - + drop_index_size); + rowset_meta->set_index_disk_size(input_rowset_meta->index_disk_size() - + drop_index_size); + } else { + rowset_meta->set_total_disk_size(input_rowset_meta->total_disk_size()); + rowset_meta->set_data_disk_size(input_rowset_meta->data_disk_size()); + rowset_meta->set_index_disk_size(input_rowset_meta->index_disk_size()); + } + } else { + for (int seg_id = 0; seg_id < num_segments; seg_id++) { + auto segment_full_path = io::Path( + static_cast(input_rowset.get())->segment_file_path(seg_id)); + std::string segment_filename = segment_full_path.filename().native(); + auto segment_dir = segment_full_path.parent_path(); + auto idx_file_reader = std::make_unique( + context.fs, segment_dir, segment_filename, + output_rs_tablet_schema->get_inverted_index_storage_format()); + auto st = idx_file_reader->init(); + if (!st.ok() && !st.is()) { + return st; + } + _inverted_index_file_readers.emplace( + std::make_pair(output_rs_writer->rowset_id().to_string(), seg_id), + std::move(idx_file_reader)); + } + rowset_meta->set_total_disk_size(input_rowset_meta->total_disk_size() - + total_index_size); + rowset_meta->set_data_disk_size(input_rowset_meta->data_disk_size() - total_index_size); + rowset_meta->set_index_disk_size(input_rowset_meta->index_disk_size() - + total_index_size); + } rowset_meta->set_empty(input_rowset_meta->empty()); rowset_meta->set_num_segments(input_rowset_meta->num_segments()); rowset_meta->set_segments_overlap(input_rowset_meta->segments_overlap()); @@ -172,7 +224,72 @@ Status IndexBuilder::update_inverted_index_info() { Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta, std::vector& segments) { if (_is_drop_op) { - // delete invertd index file by gc thread when gc input rowset + auto output_rs_tablet_schema = output_rowset_meta->tablet_schema(); + if (output_rs_tablet_schema->get_inverted_index_storage_format() != + InvertedIndexStorageFormatPB::V1) { + std::string segment_dir = _tablet->tablet_path(); + auto fs = output_rowset_meta->fs(); + auto output_rowset_schema = output_rowset_meta->tablet_schema(); + size_t inverted_index_size = 0; + for (auto& seg_ptr : segments) { + std::string segment_filename = fmt::format( + "{}_{}.dat", output_rowset_meta->rowset_id().to_string(), seg_ptr->id()); + + auto idx_file_reader_iter = _inverted_index_file_readers.find( + std::make_pair(output_rowset_meta->rowset_id().to_string(), seg_ptr->id())); + if (idx_file_reader_iter == _inverted_index_file_readers.end()) { + LOG(ERROR) << "idx_file_reader_iter" << output_rowset_meta->rowset_id() << ":" + << seg_ptr->id() << " cannot be found"; + continue; + } + auto dirs = DORIS_TRY(idx_file_reader_iter->second->get_all_directories()); + auto inverted_index_file_writer = std::make_unique( + fs, segment_dir, segment_filename, + output_rowset_schema->get_inverted_index_storage_format()); + RETURN_IF_ERROR(inverted_index_file_writer->initialize(dirs)); + // create inverted index writer + for (const auto& t_inverted_index : _alter_inverted_indexes) { + DCHECK_EQ(t_inverted_index.columns.size(), 1); + auto column_name = t_inverted_index.columns[0]; + auto column_idx = output_rs_tablet_schema->field_index(column_name); + if (column_idx < 0) { + LOG(WARNING) << "referenced column was missing. " + << "[column=" << column_name + << " referenced_column=" << column_idx << "]"; + continue; + } + auto column = output_rs_tablet_schema->column(column_idx); + const auto* index_meta = output_rs_tablet_schema->get_inverted_index(column); + if (index_meta == nullptr) { + LOG(ERROR) << "failed to find column: " << column_name + << " index_id: " << t_inverted_index.index_id; + continue; + } + RETURN_IF_ERROR(inverted_index_file_writer->delete_index(index_meta)); + } + _inverted_index_file_writers.emplace(seg_ptr->id(), + std::move(inverted_index_file_writer)); + } + for (auto& kv : _inverted_index_file_writers) { + auto* inverted_index_writer = kv.second.get(); + LOG(INFO) << "close inverted index file " + << inverted_index_writer->get_index_file_name(); + auto st = inverted_index_writer->close(); + if (!st.ok()) { + LOG(ERROR) << "close inverted index file " + << inverted_index_writer->get_index_file_name() << " error:" << st; + } + inverted_index_size += inverted_index_writer->get_index_file_size(); + } + _inverted_index_file_writers.clear(); + output_rowset_meta->set_data_disk_size(output_rowset_meta->data_disk_size() + + inverted_index_size); + output_rowset_meta->set_total_disk_size(output_rowset_meta->total_disk_size() + + inverted_index_size); + output_rowset_meta->set_index_disk_size(output_rowset_meta->index_disk_size() + + inverted_index_size); + } + LOG(INFO) << "all row nums. source_rows=" << output_rowset_meta->num_rows(); return Status::OK(); } else { // create inverted index writer @@ -186,6 +303,27 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta std::vector return_columns; std::vector> inverted_index_writer_signs; _olap_data_convertor->reserve(_alter_inverted_indexes.size()); + + std::unique_ptr inverted_index_file_writer = nullptr; + if (output_rowset_schema->get_inverted_index_storage_format() != + InvertedIndexStorageFormatPB::V1) { + auto idx_file_reader_iter = _inverted_index_file_readers.find( + std::make_pair(output_rowset_meta->rowset_id().to_string(), seg_ptr->id())); + if (idx_file_reader_iter == _inverted_index_file_readers.end()) { + LOG(ERROR) << "idx_file_reader_iter" << output_rowset_meta->rowset_id() << ":" + << seg_ptr->id() << " cannot be found"; + continue; + } + auto dirs = DORIS_TRY(idx_file_reader_iter->second->get_all_directories()); + inverted_index_file_writer = std::make_unique( + fs, segment_dir, segment_filename, + output_rowset_schema->get_inverted_index_storage_format()); + RETURN_IF_ERROR(inverted_index_file_writer->initialize(dirs)); + } else { + inverted_index_file_writer = std::make_unique( + fs, segment_dir, segment_filename, + output_rowset_schema->get_inverted_index_storage_format()); + } // create inverted index writer for (auto i = 0; i < _alter_inverted_indexes.size(); ++i) { auto inverted_index = _alter_inverted_indexes[i]; @@ -204,12 +342,12 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta _olap_data_convertor->add_column_data_convertor(column); return_columns.emplace_back(column_idx); std::unique_ptr field(FieldFactory::create(column)); - auto index_meta = output_rowset_schema->get_inverted_index(column); + const auto* index_meta = output_rowset_schema->get_inverted_index(column); std::unique_ptr inverted_index_builder; try { RETURN_IF_ERROR(segment_v2::InvertedIndexColumnWriter::create( - field.get(), &inverted_index_builder, segment_filename, segment_dir, - index_meta, fs)); + field.get(), &inverted_index_builder, inverted_index_file_writer.get(), + index_meta)); } catch (const std::exception& e) { return Status::Error( "CLuceneError occured: {}", e.what()); @@ -222,6 +360,8 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta inverted_index_writer_signs.push_back(writer_sign); } } + _inverted_index_file_writers.emplace(seg_ptr->id(), + std::move(inverted_index_file_writer)); // create iterator for each segment StorageReadOptions read_options; @@ -241,14 +381,14 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta std::shared_ptr block = std::make_shared( output_rowset_schema->create_block(return_columns)); while (true) { - auto st = iter->next_batch(block.get()); - if (!st.ok()) { - if (st.is()) { + auto status = iter->next_batch(block.get()); + if (!status.ok()) { + if (status.is()) { break; } LOG(WARNING) << "failed to read next block when schema change for inverted index." - << ", err=" << st.to_string(); + << ", err=" << status.to_string(); } // write inverted index data @@ -264,18 +404,29 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta for (auto& writer_sign : inverted_index_writer_signs) { try { if (_inverted_index_builders[writer_sign]) { - static_cast(_inverted_index_builders[writer_sign]->finish()); + RETURN_IF_ERROR(_inverted_index_builders[writer_sign]->finish()); } } catch (const std::exception& e) { return Status::Error( "CLuceneError occured: {}", e.what()); } - inverted_index_size += _inverted_index_builders[writer_sign]->file_size(); } _olap_data_convertor->reset(); } + for (auto& kv : _inverted_index_file_writers) { + auto* inverted_index_file_writer = kv.second.get(); + LOG(INFO) << "close inverted index file " + << inverted_index_file_writer->get_index_file_name(); + auto st = inverted_index_file_writer->close(); + if (!st.ok()) { + LOG(ERROR) << "close inverted index file " + << inverted_index_file_writer->get_index_file_name() << " error:" << st; + } + inverted_index_size += inverted_index_file_writer->get_index_file_size(); + } _inverted_index_builders.clear(); + _inverted_index_file_writers.clear(); output_rowset_meta->set_data_disk_size(output_rowset_meta->data_disk_size() + inverted_index_size); output_rowset_meta->set_total_disk_size(output_rowset_meta->total_disk_size() + @@ -346,11 +497,11 @@ Status IndexBuilder::_add_nullable(const std::string& column_name, if (field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { DCHECK(field->get_sub_field_count() == 1); // [size, offset_ptr, item_data_ptr, item_nullmap_ptr] - auto data_ptr = reinterpret_cast(*ptr); + const auto* data_ptr = reinterpret_cast(*ptr); // total number length - size_t element_cnt = size_t((unsigned long)(*data_ptr)); + auto element_cnt = size_t((unsigned long)(*data_ptr)); auto offset_data = *(data_ptr + 1); - const uint8_t* offsets_ptr = (const uint8_t*)offset_data; + const auto* offsets_ptr = (const uint8_t*)offset_data; try { if (element_cnt > 0) { auto data = *(data_ptr + 2); @@ -393,9 +544,9 @@ Status IndexBuilder::_add_data(const std::string& column_name, if (field->type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { DCHECK(field->get_sub_field_count() == 1); // [size, offset_ptr, item_data_ptr, item_nullmap_ptr] - auto data_ptr = reinterpret_cast(*ptr); + const auto* data_ptr = reinterpret_cast(*ptr); // total number length - size_t element_cnt = size_t((unsigned long)(*data_ptr)); + auto element_cnt = size_t((unsigned long)(*data_ptr)); auto offset_data = *(data_ptr + 1); const uint8_t* offsets_ptr = (const uint8_t*)offset_data; if (element_cnt > 0) { diff --git a/be/src/olap/task/index_builder.h b/be/src/olap/task/index_builder.h index 7baefbc035..1ed1688631 100644 --- a/be/src/olap/task/index_builder.h +++ b/be/src/olap/task/index_builder.h @@ -29,6 +29,10 @@ #include "vec/olap/olap_data_convertor.h" namespace doris { +namespace segment_v2 { +class InvertedIndexFileWriter; +class InvertedIndexFileReader; +} // namespace segment_v2 class RowsetWriter; @@ -75,6 +79,11 @@ private: std::unordered_map, std::unique_ptr> _inverted_index_builders; + std::unordered_map> + _inverted_index_file_writers; + // + std::unordered_map, std::unique_ptr> + _inverted_index_file_readers; }; using IndexBuilderSharedPtr = std::shared_ptr; diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index 181a4c90da..7a37d61326 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -1544,6 +1544,7 @@ void PInternalServiceImpl::request_slave_tablet_pull_rowset( << rowset_meta->rowset_id() << ", tablet_id=" << rowset_meta->tablet_id() << ", txn_id=" << rowset_meta->txn_id(); + auto tablet_scheme = rowset_meta->tablet_schema(); for (auto& segment : segments_size) { uint64_t file_size = segment.second; uint64_t estimate_timeout = file_size / config::download_low_speed_limit_kbps / 1024; @@ -1581,15 +1582,27 @@ void PInternalServiceImpl::request_slave_tablet_pull_rowset( auto index_id = index_size.indexid(); auto size = index_size.size(); auto suffix_path = index_size.suffix_path(); - std::string remote_inverted_index_file = - InvertedIndexDescriptor::get_index_file_name(remote_file_path, index_id, - suffix_path); - std::string remote_inverted_index_file_url = construct_url( - get_host_port(host, http_port), token, remote_inverted_index_file); + std::string remote_inverted_index_file; + std::string local_inverted_index_file; + std::string remote_inverted_index_file_url; + if (tablet_scheme->get_inverted_index_storage_format() != + InvertedIndexStorageFormatPB::V1) { + remote_inverted_index_file = + InvertedIndexDescriptor::get_index_file_name(remote_file_path); + remote_inverted_index_file_url = construct_url( + get_host_port(host, http_port), token, remote_inverted_index_file); - std::string local_inverted_index_file = - InvertedIndexDescriptor::get_index_file_name(local_file_path, index_id, - suffix_path); + local_inverted_index_file = + InvertedIndexDescriptor::get_index_file_name(local_file_path); + } else { + remote_inverted_index_file = InvertedIndexDescriptor::get_index_file_name( + remote_file_path, index_id, suffix_path); + remote_inverted_index_file_url = construct_url( + get_host_port(host, http_port), token, remote_inverted_index_file); + + local_inverted_index_file = InvertedIndexDescriptor::get_index_file_name( + local_file_path, index_id, suffix_path); + } st = download_file_action(remote_inverted_index_file_url, local_inverted_index_file, estimate_timeout, size); if (!st.ok()) { @@ -1603,6 +1616,7 @@ void PInternalServiceImpl::request_slave_tablet_pull_rowset( rowset_meta->tablet_id(), node_id, false); return; } + VLOG_CRITICAL << "succeed to download inverted index file for slave replica. url=" << remote_inverted_index_file_url diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 3b13de1a15..b9ce7e5f98 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2505,6 +2505,12 @@ public class Config extends ConfigBase { options = {"default", "ranger-doris"}) public static String access_controller_type = "default"; + @ConfField(mutable = true, masterOnly = true, description = { + "倒排索引默认存储格式", + "Default storage format of inverted index, the default value is V1." + }) + public static String inverted_index_storage_format = "V1"; + //========================================================================== // begin of cloud config //========================================================================== diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 19b2f132d2..55c939a466 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -92,6 +92,7 @@ import org.apache.doris.task.AgentTaskExecutor; import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.ClearAlterTask; import org.apache.doris.task.UpdateTabletMetaInfoTask; +import org.apache.doris.thrift.TInvertedIndexStorageFormat; import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TTaskType; @@ -1312,6 +1313,9 @@ public class SchemaChangeHandler extends AlterHandler { TStorageFormat storageFormat = PropertyAnalyzer.analyzeStorageFormat(propertyMap); + TInvertedIndexStorageFormat invertedIndexStorageFormat = + PropertyAnalyzer.analyzeInvertedIndexStorageFormat(propertyMap); + // begin checking each table // ATTN: DO NOT change any meta in this loop long tableId = olapTable.getId(); @@ -1377,6 +1381,10 @@ public class SchemaChangeHandler extends AlterHandler { if (olapTable.getStorageFormat() != TStorageFormat.V2) { needAlter = true; } + } else if (invertedIndexStorageFormat == TInvertedIndexStorageFormat.V2) { + if (olapTable.getInvertedIndexStorageFormat() != TInvertedIndexStorageFormat.V2) { + needAlter = true; + } } if (!needAlter) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java index 3ed694b998..3dc3fa24aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStmt.java @@ -37,6 +37,7 @@ public class ShowTableStmt extends ShowStmt { private static final String NAME_COL_PREFIX = "Tables_in_"; private static final String TYPE_COL = "Table_type"; private static final String STORAGE_FORMAT_COL = "Storage_format"; + private static final String INVERTED_INDEX_STORAGE_FORMAT_COL = "Inverted_index_storage_format"; private String db; private String catalog; private boolean isVerbose; @@ -164,6 +165,7 @@ public class ShowTableStmt extends ShowStmt { if (isVerbose) { builder.addColumn(new Column(TYPE_COL, ScalarType.createVarchar(20))); builder.addColumn(new Column(STORAGE_FORMAT_COL, ScalarType.createVarchar(20))); + builder.addColumn(new Column(INVERTED_INDEX_STORAGE_FORMAT_COL, ScalarType.createVarchar(20))); } return builder.build(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index effd7b6ca6..5bf2f3eb1b 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3347,6 +3347,10 @@ public class Env { sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT).append("\" = \""); sb.append(olapTable.getStorageFormat()).append("\""); + // inverted index storage type + sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT).append("\" = \""); + sb.append(olapTable.getInvertedIndexStorageFormat()).append("\""); + // compression type if (olapTable.getCompressionType() != TCompressionType.LZ4F) { sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_COMPRESSION).append("\" = \""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index dfb0d8bcd9..0badc90fd3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -65,6 +65,7 @@ import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TCompressionType; import org.apache.doris.thrift.TFetchOption; +import org.apache.doris.thrift.TInvertedIndexStorageFormat; import org.apache.doris.thrift.TOlapTable; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.TSortType; @@ -2319,6 +2320,13 @@ public class OlapTable extends Table implements MTMVRelatedTableIf { tableProperty.buildStorageFormat(); } + public void setInvertedIndexStorageFormat(TInvertedIndexStorageFormat invertedIndexStorageFormat) { + TableProperty tableProperty = getOrCreatTableProperty(); + tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, + invertedIndexStorageFormat.name()); + tableProperty.buildInvertedIndexStorageFormat(); + } + public TStorageFormat getStorageFormat() { if (tableProperty == null) { return TStorageFormat.DEFAULT; @@ -2326,6 +2334,13 @@ public class OlapTable extends Table implements MTMVRelatedTableIf { return tableProperty.getStorageFormat(); } + public TInvertedIndexStorageFormat getInvertedIndexStorageFormat() { + if (tableProperty == null) { + return TInvertedIndexStorageFormat.V2; + } + return tableProperty.getInvertedIndexStorageFormat(); + } + public TCompressionType getCompressionType() { if (tableProperty == null) { return TCompressionType.LZ4F; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index 3d5215bc48..dd3dd165d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -26,6 +26,7 @@ import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.persist.OperationType; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.thrift.TCompressionType; +import org.apache.doris.thrift.TInvertedIndexStorageFormat; import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; @@ -77,6 +78,8 @@ public class TableProperty implements Writable { */ private TStorageFormat storageFormat = TStorageFormat.DEFAULT; + private TInvertedIndexStorageFormat invertedIndexStorageFormat = TInvertedIndexStorageFormat.DEFAULT; + private TCompressionType compressionType = TCompressionType.LZ4F; private boolean enableLightSchemaChange = false; @@ -416,6 +419,13 @@ public class TableProperty implements Writable { return this; } + public TableProperty buildInvertedIndexStorageFormat() { + invertedIndexStorageFormat = TInvertedIndexStorageFormat.valueOf(properties.getOrDefault( + PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, + TInvertedIndexStorageFormat.DEFAULT.name())); + return this; + } + public void modifyTableProperties(Map modifyProperties) { properties.putAll(modifyProperties); removeDuplicateReplicaNumProperty(); @@ -479,6 +489,10 @@ public class TableProperty implements Writable { return storageFormat; } + public TInvertedIndexStorageFormat getInvertedIndexStorageFormat() { + return invertedIndexStorageFormat; + } + public DataSortInfo getDataSortInfo() { return dataSortInfo; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index a09694ada1..9bacb50db3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -40,6 +40,7 @@ import org.apache.doris.policy.StoragePolicy; import org.apache.doris.resource.Tag; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TCompressionType; +import org.apache.doris.thrift.TInvertedIndexStorageFormat; import org.apache.doris.thrift.TSortType; import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; @@ -99,6 +100,8 @@ public class PropertyAnalyzer { */ public static final String PROPERTIES_STORAGE_FORMAT = "storage_format"; + public static final String PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT = "inverted_index_storage_format"; + public static final String PROPERTIES_INMEMORY = "in_memory"; // _auto_bucket can only set in create table stmt rewrite bucket and can not be changed @@ -873,6 +876,35 @@ public class PropertyAnalyzer { } } + public static TInvertedIndexStorageFormat analyzeInvertedIndexStorageFormat(Map properties) + throws AnalysisException { + String invertedIndexStorageFormat = ""; + if (properties != null && properties.containsKey(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT)) { + invertedIndexStorageFormat = properties.get(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT); + properties.remove(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT); + } else { + if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { + return TInvertedIndexStorageFormat.V1; + } else { + return TInvertedIndexStorageFormat.V2; + } + } + + if (invertedIndexStorageFormat.equalsIgnoreCase("v1")) { + return TInvertedIndexStorageFormat.V1; + } else if (invertedIndexStorageFormat.equalsIgnoreCase("v2")) { + return TInvertedIndexStorageFormat.V2; + } else if (invertedIndexStorageFormat.equalsIgnoreCase("default")) { + if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { + return TInvertedIndexStorageFormat.V1; + } else { + return TInvertedIndexStorageFormat.V2; + } + } else { + throw new AnalysisException("unknown inverted index storage format: " + invertedIndexStorageFormat); + } + } + // analyze common boolean properties, such as "in_memory" = "false" public static boolean analyzeBooleanProp(Map properties, String propKey, boolean defaultVal) { if (properties != null && properties.containsKey(propKey)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index afb3d73f35..b6b89c0ad7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -160,6 +160,7 @@ import org.apache.doris.task.AgentTaskExecutor; import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.CreateReplicaTask; import org.apache.doris.thrift.TCompressionType; +import org.apache.doris.thrift.TInvertedIndexStorageFormat; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; @@ -1832,7 +1833,6 @@ public class InternalCatalog implements CatalogIf { BinlogConfig binlogConfig, boolean isStorageMediumSpecified, List clusterKeyIndexes) throws DdlException { - // create base index first. Preconditions.checkArgument(tbl.getBaseIndexId() != -1); MaterializedIndex baseIndex = new MaterializedIndex(tbl.getBaseIndexId(), IndexState.NORMAL); @@ -1912,6 +1912,7 @@ public class InternalCatalog implements CatalogIf { tbl.storeRowColumn(), binlogConfig); task.setStorageFormat(tbl.getStorageFormat()); + task.setInvertedIndexStorageFormat(tbl.getInvertedIndexStorageFormat()); task.setClusterKeyIndexes(clusterKeyIndexes); batchTask.addTask(task); // add to AgentTaskQueue for handling finish report. @@ -2232,6 +2233,14 @@ public class InternalCatalog implements CatalogIf { } olapTable.setStorageFormat(storageFormat); + TInvertedIndexStorageFormat invertedIndexStorageFormat; + try { + invertedIndexStorageFormat = PropertyAnalyzer.analyzeInvertedIndexStorageFormat(properties); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + olapTable.setInvertedIndexStorageFormat(invertedIndexStorageFormat); + // get compression type TCompressionType compressionType = TCompressionType.LZ4; try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 8bafeaae28..ffd13e00eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -873,10 +873,13 @@ public class ShowExecutor { } if (showTableStmt.isVerbose()) { String storageFormat = "NONE"; + String invertedIndexStorageFormat = "NONE"; if (tbl instanceof OlapTable) { storageFormat = ((OlapTable) tbl).getStorageFormat().toString(); + invertedIndexStorageFormat = ((OlapTable) tbl).getInvertedIndexStorageFormat().toString(); } - rows.add(Lists.newArrayList(tbl.getName(), tbl.getMysqlType(), storageFormat)); + rows.add(Lists.newArrayList(tbl.getName(), tbl.getMysqlType(), storageFormat, + invertedIndexStorageFormat)); } else { rows.add(Lists.newArrayList(tbl.getName())); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 476de574b6..262ac8e84f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -31,6 +31,7 @@ import org.apache.doris.policy.PolicyTypeEnum; import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TCompressionType; import org.apache.doris.thrift.TCreateTabletReq; +import org.apache.doris.thrift.TInvertedIndexStorageFormat; import org.apache.doris.thrift.TOlapTableIndex; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TStorageFormat; @@ -89,6 +90,8 @@ public class CreateReplicaTask extends AgentTask { // TODO should unify the naming of v1(alpha rowset), v2(beta rowset), it is very confused to read code private TStorageFormat storageFormat = TStorageFormat.V2; + private TInvertedIndexStorageFormat invertedIndexStorageFormat = TInvertedIndexStorageFormat.V1; + // true if this task is created by recover request(See comment of Config.recover_with_empty_tablet) private boolean isRecoverTask = false; @@ -229,6 +232,10 @@ public class CreateReplicaTask extends AgentTask { this.storageFormat = storageFormat; } + public void setInvertedIndexStorageFormat(TInvertedIndexStorageFormat invertedIndexStorageFormat) { + this.invertedIndexStorageFormat = invertedIndexStorageFormat; + } + public void setClusterKeyIndexes(List clusterKeyIndexes) { this.clusterKeyIndexes = clusterKeyIndexes; } @@ -325,6 +332,9 @@ public class CreateReplicaTask extends AgentTask { createTabletReq.setStorageFormat(storageFormat); } + if (invertedIndexStorageFormat != null) { + createTabletReq.setInvertedIndexStorageFormat(invertedIndexStorageFormat); + } createTabletReq.setTabletType(tabletType); createTabletReq.setCompressionType(compressionType); createTabletReq.setEnableUniqueKeyMergeOnWrite(enableUniqueKeyMergeOnWrite); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 13811d3af4..ca4233e17c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -96,6 +96,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -121,6 +122,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -142,6 +144,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -181,6 +184,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -209,6 +213,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -239,6 +244,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -266,6 +272,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -289,6 +296,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -319,6 +327,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -346,6 +355,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -377,6 +387,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -405,6 +416,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -432,6 +444,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -455,6 +468,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -485,6 +499,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -514,6 +529,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -542,6 +558,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -571,6 +588,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"enable_unique_key_merge_on_write\" = \"true\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" @@ -625,6 +643,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -646,6 +665,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -681,6 +701,7 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" + + "\"inverted_index_storage_format\" = \"V1\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowTableStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowTableStmtTest.java index ea197c9b36..cd72b41baf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowTableStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowTableStmtTest.java @@ -55,7 +55,7 @@ public class ShowTableStmtTest { stmt = new ShowTableStmt("abc", null, true, null); stmt.analyze(analyzer); Assert.assertEquals("SHOW FULL TABLES FROM internal.abc", stmt.toString()); - Assert.assertEquals(3, stmt.getMetaData().getColumnCount()); + Assert.assertEquals(4, stmt.getMetaData().getColumnCount()); Assert.assertEquals("Tables_in_abc", stmt.getMetaData().getColumn(0).getName()); Assert.assertEquals("Table_type", stmt.getMetaData().getColumn(1).getName()); @@ -63,7 +63,7 @@ public class ShowTableStmtTest { stmt.analyze(analyzer); Assert.assertEquals("bcd", stmt.getPattern()); Assert.assertEquals("SHOW FULL TABLES FROM internal.abc LIKE 'bcd'", stmt.toString()); - Assert.assertEquals(3, stmt.getMetaData().getColumnCount()); + Assert.assertEquals(4, stmt.getMetaData().getColumnCount()); Assert.assertEquals("Tables_in_abc", stmt.getMetaData().getColumn(0).getName()); Assert.assertEquals("Table_type", stmt.getMetaData().getColumn(1).getName()); } diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index f14ba23564..5eedd8af07 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -220,6 +220,11 @@ enum IndexType { NGRAM_BF = 3; } +enum InvertedIndexStorageFormatPB { + V1 = 0; + V2 = 1; +} + message TabletIndexPB { optional int64 index_id = 1; optional string index_name = 2; @@ -259,6 +264,7 @@ message TabletSchemaPB { optional bool enable_single_replica_compaction = 22 [default=false]; optional bool skip_write_index_on_load = 23 [default=false]; repeated int32 cluster_key_idxes = 24; + optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25 [default=V1]; } enum TabletStatePB { diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index a1c4020b14..99196c2e02 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -111,6 +111,14 @@ enum TCompressionType { LZ4HC = 8 } +// Enumerates the storage formats for inverted indexes in src_backends. +// This enum is used to distinguish between different organizational methods +// of inverted index data, affecting how the index is stored and accessed. +enum TInvertedIndexStorageFormat { + DEFAULT, // Default format, unspecified storage method. + V1, // Index per idx: Each index is stored separately based on its identifier. + V2 // Segment id per idx: Indexes are organized based on segment identifiers, grouping indexes by their associated segment. +} struct TBinlogConfig { 1: optional bool enable; @@ -151,6 +159,7 @@ struct TCreateTabletReq { 24: optional i64 time_series_compaction_file_count_threshold = 2000 25: optional i64 time_series_compaction_time_threshold_seconds = 3600 26: optional i64 time_series_compaction_empty_rowsets_threshold = 5 + 27: optional TInvertedIndexStorageFormat inverted_index_storage_format = TInvertedIndexStorageFormat.V1 } struct TDropTabletReq { diff --git a/regression-test/data/inverted_index_p0/index_change/test_index_change_1.out b/regression-test/data/inverted_index_p0/index_change/test_index_change_1.out index 008afcb51e..fb32ebba5e 100644 --- a/regression-test/data/inverted_index_p0/index_change/test_index_change_1.out +++ b/regression-test/data/inverted_index_p0/index_change/test_index_change_1.out @@ -24,3 +24,28 @@ -- !select6 -- 3 2017-10-01 Shanghai China 10 1 electrical engineer +-- !select1_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +3 2017-10-01 Shanghai China 10 1 electrical engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist +5 2017-10-02 Shenzhen China 10 1 teacher +6 2017-10-03 Hongkong China 10 1 Architectural designer + +-- !select2_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist + +-- !select3_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer + +-- !select4_v1 -- + +-- !select5_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select6_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + diff --git a/regression-test/data/inverted_index_p0/index_change/test_index_change_2.out b/regression-test/data/inverted_index_p0/index_change/test_index_change_2.out index ab3e0c076c..bbfc3579ae 100644 --- a/regression-test/data/inverted_index_p0/index_change/test_index_change_2.out +++ b/regression-test/data/inverted_index_p0/index_change/test_index_change_2.out @@ -41,3 +41,45 @@ -- !select11 -- 3 2017-10-01 Shanghai China 10 1 electrical engineer +-- !select1_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +3 2017-10-01 Shanghai China 10 1 electrical engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist +5 2017-10-02 Shenzhen China 10 1 teacher +6 2017-10-03 Hongkong China 10 1 Architectural designer + +-- !select2_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist + +-- !select3_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer + +-- !select4_v1 -- + +-- !select5_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select6_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select7_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist + +-- !select8_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer + +-- !select9_v1 -- + +-- !select10_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select11_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + diff --git a/regression-test/data/inverted_index_p0/index_change/test_index_change_3.out b/regression-test/data/inverted_index_p0/index_change/test_index_change_3.out index ab3e0c076c..bbfc3579ae 100644 --- a/regression-test/data/inverted_index_p0/index_change/test_index_change_3.out +++ b/regression-test/data/inverted_index_p0/index_change/test_index_change_3.out @@ -41,3 +41,45 @@ -- !select11 -- 3 2017-10-01 Shanghai China 10 1 electrical engineer +-- !select1_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +3 2017-10-01 Shanghai China 10 1 electrical engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist +5 2017-10-02 Shenzhen China 10 1 teacher +6 2017-10-03 Hongkong China 10 1 Architectural designer + +-- !select2_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist + +-- !select3_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer + +-- !select4_v1 -- + +-- !select5_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select6_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select7_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist + +-- !select8_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer + +-- !select9_v1 -- + +-- !select10_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select11_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + diff --git a/regression-test/data/inverted_index_p0/index_change/test_index_change_4.out b/regression-test/data/inverted_index_p0/index_change/test_index_change_4.out index ab3e0c076c..bbfc3579ae 100644 --- a/regression-test/data/inverted_index_p0/index_change/test_index_change_4.out +++ b/regression-test/data/inverted_index_p0/index_change/test_index_change_4.out @@ -41,3 +41,45 @@ -- !select11 -- 3 2017-10-01 Shanghai China 10 1 electrical engineer +-- !select1_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +3 2017-10-01 Shanghai China 10 1 electrical engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist +5 2017-10-02 Shenzhen China 10 1 teacher +6 2017-10-03 Hongkong China 10 1 Architectural designer + +-- !select2_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist + +-- !select3_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer + +-- !select4_v1 -- + +-- !select5_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select6_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select7_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist + +-- !select8_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer + +-- !select9_v1 -- + +-- !select10_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select11_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + diff --git a/regression-test/data/inverted_index_p0/index_change/test_index_change_5.out b/regression-test/data/inverted_index_p0/index_change/test_index_change_5.out index ab3e0c076c..bbfc3579ae 100644 --- a/regression-test/data/inverted_index_p0/index_change/test_index_change_5.out +++ b/regression-test/data/inverted_index_p0/index_change/test_index_change_5.out @@ -41,3 +41,45 @@ -- !select11 -- 3 2017-10-01 Shanghai China 10 1 electrical engineer +-- !select1_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +3 2017-10-01 Shanghai China 10 1 electrical engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist +5 2017-10-02 Shenzhen China 10 1 teacher +6 2017-10-03 Hongkong China 10 1 Architectural designer + +-- !select2_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist + +-- !select3_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer + +-- !select4_v1 -- + +-- !select5_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select6_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select7_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist + +-- !select8_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer + +-- !select9_v1 -- + +-- !select10_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select11_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + diff --git a/regression-test/data/inverted_index_p0/index_change/test_index_change_6.out b/regression-test/data/inverted_index_p0/index_change/test_index_change_6.out index ab3e0c076c..bbfc3579ae 100644 --- a/regression-test/data/inverted_index_p0/index_change/test_index_change_6.out +++ b/regression-test/data/inverted_index_p0/index_change/test_index_change_6.out @@ -41,3 +41,45 @@ -- !select11 -- 3 2017-10-01 Shanghai China 10 1 electrical engineer +-- !select1_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +3 2017-10-01 Shanghai China 10 1 electrical engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist +5 2017-10-02 Shenzhen China 10 1 teacher +6 2017-10-03 Hongkong China 10 1 Architectural designer + +-- !select2_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist + +-- !select3_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer + +-- !select4_v1 -- + +-- !select5_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select6_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select7_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer +4 2017-10-02 Beijing China 10 0 Both a teacher and a scientist + +-- !select8_v1 -- +1 2017-10-01 Beijing China 10 1 Software Developer +2 2017-10-01 Beijing China 10 1 Communication Engineer + +-- !select9_v1 -- + +-- !select10_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + +-- !select11_v1 -- +3 2017-10-01 Shanghai China 10 1 electrical engineer + diff --git a/regression-test/data/inverted_index_p0/storage_format/documents-1000.json b/regression-test/data/inverted_index_p0/storage_format/documents-1000.json new file mode 100644 index 0000000000..a1b0aa8342 --- /dev/null +++ b/regression-test/data/inverted_index_p0/storage_format/documents-1000.json @@ -0,0 +1,1000 @@ +{"@timestamp": 893964617, "clientip":"40.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964653, "clientip":"232.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964672, "clientip":"26.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964679, "clientip":"247.37.0.0", "request": "GET /french/splash_inet.html HTTP/1.0", "status": 200, "size": 3781} +{"@timestamp": 893964682, "clientip":"247.37.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964687, "clientip":"252.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964689, "clientip":"247.37.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964689, "clientip":"247.37.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964692, "clientip":"247.37.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893964703, "clientip":"247.37.0.0", "request": "GET /french/images/nav_venue_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964704, "clientip":"247.37.0.0", "request": "GET /french/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 1139} +{"@timestamp": 893964712, "clientip":"2.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964712, "clientip":"247.37.0.0", "request": "GET /french/tickets/body.html HTTP/1.0", "status": 200, "size": 3029} +{"@timestamp": 893964726, "clientip":"120.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964736, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_hm_header.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964736, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_hm_nav.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964737, "clientip":"247.37.0.0", "request": "GET /images/arw_lk.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964750, "clientip":"247.37.0.0", "request": "GET /french/tickets/tck_0804.htm HTTP/1.0", "status": 200, "size": 14521} +{"@timestamp": 893964753, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_quest_bg2.jpg HTTP/1.0", "status": 200, "size": 11324} +{"@timestamp": 893964755, "clientip":"126.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964758, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_header.gif HTTP/1.0", "status": 200, "size": 671} +{"@timestamp": 893964758, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_bu_abroad2.gif HTTP/1.0", "status": 200, "size": 1512} +{"@timestamp": 893964758, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_bu_infrance2.gif HTTP/1.0", "status": 200, "size": 1136} +{"@timestamp": 893964758, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 1647} +{"@timestamp": 893964758, "clientip":"247.37.0.0", "request": "GET /french/tickets/images/ticket_bu_quest2.gif HTTP/1.0", "status": 200, "size": 1271} +{"@timestamp": 893964772, "clientip":"247.37.0.0", "request": "GET /french/news/3004bres.htm HTTP/1.0", "status": 200, "size": 5933} +{"@timestamp": 893964778, "clientip":"247.37.0.0", "request": "GET /french/images/hm_f98_top.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893964779, "clientip":"247.37.0.0", "request": "GET /images/bord_d.gif HTTP/1.0", "status": 200, "size": 231} +{"@timestamp": 893964779, "clientip":"247.37.0.0", "request": "GET /images/bord_g.gif HTTP/1.0", "status": 200, "size": 231} +{"@timestamp": 893964785, "clientip":"13.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964789, "clientip":"138.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964800, "clientip":"28.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964809, "clientip":"31.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964812, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893964815, "clientip":"29.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964825, "clientip":"92.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964841, "clientip":"134.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964844, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893964864, "clientip":"137.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964932, "clientip":"167.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964936, "clientip":"174.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964961, "clientip":"97.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893964973, "clientip":"174.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965016, "clientip":"121.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965036, "clientip":"2.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965050, "clientip":"91.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965059, "clientip":"47.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965070, "clientip":"142.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965084, "clientip":"104.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965106, "clientip":"40.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965115, "clientip":"109.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965123, "clientip":"184.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965139, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965149, "clientip":"56.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965150, "clientip":"161.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965150, "clientip":"237.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965154, "clientip":"41.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965162, "clientip":"131.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965172, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965182, "clientip":"42.135.0.0", "request": "GET /fth.htm HTTP/1.1", "status": 200, "size": 190} +{"@timestamp": 893965193, "clientip":"180.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965198, "clientip":"176.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965200, "clientip":"192.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965278, "clientip":"4.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965293, "clientip":"76.6.0.0", "request": "GET /images/teams_hm_bg.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965300, "clientip":"235.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965310, "clientip":"183.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965318, "clientip":"248.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965322, "clientip":"249.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965352, "clientip":"218.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965382, "clientip":"128.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965405, "clientip":"139.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965412, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965424, "clientip":"249.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965449, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965464, "clientip":"221.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965467, "clientip":"175.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965479, "clientip":"43.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965504, "clientip":"201.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965518, "clientip":"17.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965550, "clientip":"138.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965554, "clientip":"232.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965578, "clientip":"76.6.0.0", "request": "GET /images/nantes.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965585, "clientip":"252.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965598, "clientip":"120.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965616, "clientip":"2.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965622, "clientip":"44.135.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965626, "clientip":"44.135.0.0", "request": "GET /french/nav_inet.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965626, "clientip":"44.135.0.0", "request": "GET /french/splash_inet.html HTTP/1.0", "status": 200, "size": 3781} +{"@timestamp": 893965626, "clientip":"44.135.0.0", "request": "GET /french/nav_top_inet.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965627, "clientip":"44.135.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965628, "clientip":"44.135.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 200, "size": 33665} +{"@timestamp": 893965631, "clientip":"44.135.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965632, "clientip":"44.135.0.0", "request": "GET /french/ProScroll.class HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965634, "clientip":"44.135.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965634, "clientip":"44.135.0.0", "request": "GET /french/images/nav_news_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965634, "clientip":"44.135.0.0", "request": "GET /french/images/nav_comp_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965638, "clientip":"44.135.0.0", "request": "GET /french/images/nav_venue_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965648, "clientip":"126.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965649, "clientip":"44.135.0.0", "request": "GET /french/images/nav_tickets_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965651, "clientip":"44.135.0.0", "request": "GET /french/news/3004bres.htm HTTP/1.0", "status": 200, "size": 5933} +{"@timestamp": 893965651, "clientip":"44.135.0.0", "request": "GET /french/images/nav_field_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965652, "clientip":"44.135.0.0", "request": "GET /french/images/nav_history_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965655, "clientip":"44.135.0.0", "request": "GET /images/backnews.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965658, "clientip":"44.135.0.0", "request": "GET /french/images/nav_team_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965658, "clientip":"44.135.0.0", "request": "GET /french/images/nav_store_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965658, "clientip":"44.135.0.0", "request": "GET /french/images/nav_home_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965659, "clientip":"44.135.0.0", "request": "GET /french/images/nav_sitemap_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965659, "clientip":"44.135.0.0", "request": "GET /french/images/fpnewstop.gif HTTP/1.0", "status": 200, "size": 1317} +{"@timestamp": 893965660, "clientip":"44.135.0.0", "request": "GET /french/images/hm_f98_top.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965661, "clientip":"44.135.0.0", "request": "GET /french/images/news_btn_letter_off.gif HTTP/1.0", "status": 200, "size": 871} +{"@timestamp": 893965661, "clientip":"44.135.0.0", "request": "GET /french/images/news_btn_press_off.gif HTTP/1.0", "status": 200, "size": 1795} +{"@timestamp": 893965662, "clientip":"44.135.0.0", "request": "GET /french/images/news_btn_kits_off.gif HTTP/1.0", "status": 200, "size": 965} +{"@timestamp": 893965662, "clientip":"44.135.0.0", "request": "GET /french/images/nav_hosts_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965662, "clientip":"44.135.0.0", "request": "GET /french/images/nav_logo_sponsors.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965663, "clientip":"44.135.0.0", "request": "GET /images/bord_d.gif HTTP/1.0", "status": 200, "size": 231} +{"@timestamp": 893965664, "clientip":"44.135.0.0", "request": "GET /images/bord_g.gif HTTP/1.0", "status": 200, "size": 231} +{"@timestamp": 893965668, "clientip":"26.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965684, "clientip":"13.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965700, "clientip":"28.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965700, "clientip":"44.135.0.0", "request": "GET /french/news/newsprr.htm HTTP/1.0", "status": 200, "size": 28486} +{"@timestamp": 893965705, "clientip":"44.135.0.0", "request": "GET /french/images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965706, "clientip":"44.135.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965706, "clientip":"44.135.0.0", "request": "GET /french/images/news_btn_press_on.gif HTTP/1.0", "status": 200, "size": 1757} +{"@timestamp": 893965706, "clientip":"44.135.0.0", "request": "GET /images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893965708, "clientip":"44.135.0.0", "request": "GET /french/images/news_hd_press.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965708, "clientip":"44.135.0.0", "request": "GET /images/news_arrow.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893965712, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965724, "clientip":"31.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965727, "clientip":"92.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965727, "clientip":"134.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965731, "clientip":"29.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965738, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965753, "clientip":"237.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965796, "clientip":"42.135.0.0", "request": "GET /fth.htm HTTP/1.1", "status": 200, "size": 190} +{"@timestamp": 893965849, "clientip":"174.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965856, "clientip":"167.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965865, "clientip":"137.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965882, "clientip":"40.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965883, "clientip":"97.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965896, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/argentina78.html HTTP/1.0", "status": 200, "size": 20916} +{"@timestamp": 893965899, "clientip":"45.135.0.0", "request": "GET /french/history/images/history_hm_header.gif HTTP/1.0", "status": 200, "size": 1034} +{"@timestamp": 893965899, "clientip":"45.135.0.0", "request": "GET /french/history/images/football.GIF HTTP/1.0", "status": 200, "size": 1452} +{"@timestamp": 893965899, "clientip":"45.135.0.0", "request": "GET /french/history/images/france98b.GIF HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893965901, "clientip":"45.135.0.0", "request": "GET /french/history/images/infrance.GIF HTTP/1.0", "status": 200, "size": 1017} +{"@timestamp": 893965901, "clientip":"45.135.0.0", "request": "GET /french/history/images/reading.GIF HTTP/1.0", "status": 200, "size": 1094} +{"@timestamp": 893965901, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_30_off.gif HTTP/1.0", "status": 200, "size": 406} +{"@timestamp": 893965902, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_38_off.gif HTTP/1.0", "status": 200, "size": 436} +{"@timestamp": 893965903, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_54_off.gif HTTP/1.0", "status": 200, "size": 403} +{"@timestamp": 893965904, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_62_off.gif HTTP/1.0", "status": 200, "size": 437} +{"@timestamp": 893965905, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_70_off.gif HTTP/1.0", "status": 200, "size": 409} +{"@timestamp": 893965905, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_78_on.gif HTTP/1.0", "status": 200, "size": 501} +{"@timestamp": 893965906, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_94_off.gif HTTP/1.0", "status": 200, "size": 432} +{"@timestamp": 893965906, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_86_off.gif HTTP/1.0", "status": 200, "size": 409} +{"@timestamp": 893965906, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/posters/argentina78.gif HTTP/1.0", "status": 200, "size": 4033} +{"@timestamp": 893965907, "clientip":"45.135.0.0", "request": "GET /images/hist7802.jpg HTTP/1.0", "status": 200, "size": 10813} +{"@timestamp": 893965907, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_trophy2.gif HTTP/1.0", "status": 200, "size": 443} +{"@timestamp": 893965908, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_50_off.gif HTTP/1.0", "status": 200, "size": 392} +{"@timestamp": 893965908, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_header.gif HTTP/1.0", "status": 200, "size": 2077} +{"@timestamp": 893965910, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_66_off.gif HTTP/1.0", "status": 200, "size": 435} +{"@timestamp": 893965911, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_82_off.gif HTTP/1.0", "status": 200, "size": 433} +{"@timestamp": 893965912, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bracket_top.gif HTTP/1.0", "status": 200, "size": 289} +{"@timestamp": 893965913, "clientip":"45.135.0.0", "request": "GET /images/hist7801.jpg HTTP/1.0", "status": 200, "size": 12855} +{"@timestamp": 893965913, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_74_off.gif HTTP/1.0", "status": 200, "size": 423} +{"@timestamp": 893965913, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_34_off.gif HTTP/1.0", "status": 200, "size": 397} +{"@timestamp": 893965915, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bracket_bot.gif HTTP/1.0", "status": 200, "size": 631} +{"@timestamp": 893965917, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_58_off.gif HTTP/1.0", "status": 200, "size": 397} +{"@timestamp": 893965917, "clientip":"45.135.0.0", "request": "GET /french/history/past_cups/images/past_bu_90_off.gif HTTP/1.0", "status": 200, "size": 429} +{"@timestamp": 893965917, "clientip":"45.135.0.0", "request": "GET /french/history/images/thecup.GIF HTTP/1.0", "status": 200, "size": 1036} +{"@timestamp": 893965930, "clientip":"174.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965932, "clientip":"121.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965937, "clientip":"91.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965976, "clientip":"142.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965979, "clientip":"104.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965988, "clientip":"47.135.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893965991, "clientip":"47.135.0.0", "request": "GET /french/nav_top_inet.html HTTP/1.0", "status": 200, "size": 374} +{"@timestamp": 893965991, "clientip":"47.135.0.0", "request": "GET /french/splash_inet.html HTTP/1.0", "status": 200, "size": 3781} +{"@timestamp": 893965991, "clientip":"47.135.0.0", "request": "GET /french/nav_inet.html HTTP/1.0", "status": 200, "size": 2739} +{"@timestamp": 893965992, "clientip":"109.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893965993, "clientip":"47.135.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893965993, "clientip":"47.135.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893965993, "clientip":"47.135.0.0", "request": "GET /french/images/nav_team_off.gif HTTP/1.0", "status": 200, "size": 870} +{"@timestamp": 893965993, "clientip":"47.135.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893965994, "clientip":"47.135.0.0", "request": "GET /french/images/nav_news_off.gif HTTP/1.0", "status": 200, "size": 855} +{"@timestamp": 893965994, "clientip":"47.135.0.0", "request": "GET /french/images/nav_comp_off.gif HTTP/1.0", "status": 200, "size": 995} +{"@timestamp": 893965998, "clientip":"47.135.0.0", "request": "GET /french/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 413} +{"@timestamp": 893966000, "clientip":"47.135.0.0", "request": "GET /french/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 982} +{"@timestamp": 893966001, "clientip":"47.135.0.0", "request": "GET /french/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 1139} +{"@timestamp": 893966002, "clientip":"47.135.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 200, "size": 33665} +{"@timestamp": 893966002, "clientip":"47.135.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966003, "clientip":"47.135.0.0", "request": "GET /french/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 976} +{"@timestamp": 893966004, "clientip":"47.135.0.0", "request": "GET /french/ProScroll.class HTTP/1.0", "status": 200, "size": 6507} +{"@timestamp": 893966004, "clientip":"47.135.0.0", "request": "GET /french/images/hm_official.gif HTTP/1.0", "status": 200, "size": 972} +{"@timestamp": 893966004, "clientip":"47.135.0.0", "request": "GET /french/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966004, "clientip":"46.135.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966005, "clientip":"47.135.0.0", "request": "GET /french/images/nav_home_off.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966006, "clientip":"47.135.0.0", "request": "GET /french/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 966} +{"@timestamp": 893966008, "clientip":"47.135.0.0", "request": "GET /french/images/nav_venue_off.gif HTTP/1.0", "status": 200, "size": 945} +{"@timestamp": 893966010, "clientip":"47.135.0.0", "request": "GET /french/images/nav_tickets_off.gif HTTP/1.0", "status": 200, "size": 965} +{"@timestamp": 893966011, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893966017, "clientip":"47.135.0.0", "request": "GET /images/hm_anime_f.gif HTTP/1.0", "status": 200, "size": 15529} +{"@timestamp": 893966017, "clientip":"47.135.0.0", "request": "GET /images/hm_day_f.gif HTTP/1.0", "status": 200, "size": 574} +{"@timestamp": 893966019, "clientip":"47.135.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 200, "size": 208} +{"@timestamp": 893966021, "clientip":"47.135.0.0", "request": "GET /images/hm_linkf.gif HTTP/1.0", "status": 200, "size": 123} +{"@timestamp": 893966024, "clientip":"47.135.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966028, "clientip":"47.135.0.0", "request": "GET /images/dot.gif HTTP/1.0", "status": 200, "size": 43} +{"@timestamp": 893966028, "clientip":"47.135.0.0", "request": "GET /images/hm_brdr.gif HTTP/1.0", "status": 200, "size": 235} +{"@timestamp": 893966031, "clientip":"47.135.0.0", "request": "GET /images/info.gif HTTP/1.0", "status": 200, "size": 1251} +{"@timestamp": 893966048, "clientip":"131.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966055, "clientip":"41.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966068, "clientip":"161.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966068, "clientip":"56.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966084, "clientip":"176.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966085, "clientip":"47.135.0.0", "request": "GET /french/frntpage.htm HTTP/1.0", "status": 200, "size": 12824} +{"@timestamp": 893966088, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893966090, "clientip":"47.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966091, "clientip":"47.135.0.0", "request": "GET /images/backnews.gif HTTP/1.0", "status": 200, "size": 4573} +{"@timestamp": 893966091, "clientip":"47.135.0.0", "request": "GET /french/images/fpnewstop.gif HTTP/1.0", "status": 200, "size": 1317} +{"@timestamp": 893966091, "clientip":"47.135.0.0", "request": "GET /french/images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966092, "clientip":"180.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966096, "clientip":"47.135.0.0", "request": "GET /images/ligne1_case5.gif HTTP/1.0", "status": 200, "size": 1018} +{"@timestamp": 893966097, "clientip":"47.135.0.0", "request": "GET /images/case5.gif HTTP/1.0", "status": 200, "size": 1362} +{"@timestamp": 893966097, "clientip":"47.135.0.0", "request": "GET /images/dburton.jpg HTTP/1.0", "status": 200, "size": 12009} +{"@timestamp": 893966098, "clientip":"47.135.0.0", "request": "GET /images/ligne.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966100, "clientip":"47.135.0.0", "request": "GET /images/ligneb.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966102, "clientip":"47.135.0.0", "request": "GET /images/ligneb01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966102, "clientip":"47.135.0.0", "request": "GET /images/ligne01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966104, "clientip":"47.135.0.0", "request": "GET /images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966104, "clientip":"47.135.0.0", "request": "GET /images/base.gif HTTP/1.0", "status": 200, "size": 366} +{"@timestamp": 893966108, "clientip":"47.135.0.0", "request": "GET /french/images/news_btn_press_off.gif HTTP/1.0", "status": 200, "size": 1795} +{"@timestamp": 893966109, "clientip":"47.135.0.0", "request": "GET /french/images/lateb_new.gif HTTP/1.0", "status": 200, "size": 1285} +{"@timestamp": 893966109, "clientip":"47.135.0.0", "request": "GET /french/images/today_new.gif HTTP/1.0", "status": 200, "size": 869} +{"@timestamp": 893966110, "clientip":"47.135.0.0", "request": "GET /images/bord_stories.gif HTTP/1.0", "status": 200, "size": 520} +{"@timestamp": 893966111, "clientip":"47.135.0.0", "request": "GET /images/ligne4_latebreak.gif HTTP/1.0", "status": 200, "size": 1056} +{"@timestamp": 893966112, "clientip":"47.135.0.0", "request": "GET /images/bord_stories01.gif HTTP/1.0", "status": 200, "size": 333} +{"@timestamp": 893966113, "clientip":"47.135.0.0", "request": "GET /french/images/archives.gif HTTP/1.0", "status": 200, "size": 569} +{"@timestamp": 893966113, "clientip":"47.135.0.0", "request": "GET /french/images/top_stories.gif HTTP/1.0", "status": 200, "size": 1078} +{"@timestamp": 893966113, "clientip":"47.135.0.0", "request": "GET /french/images/news_btn_letter_off.gif HTTP/1.0", "status": 200, "size": 871} +{"@timestamp": 893966115, "clientip":"47.135.0.0", "request": "GET /french/images/news_btn_kits_off.gif HTTP/1.0", "status": 200, "size": 965} +{"@timestamp": 893966135, "clientip":"184.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966178, "clientip":"4.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966210, "clientip":"183.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966218, "clientip":"249.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966220, "clientip":"235.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966221, "clientip":"47.135.0.0", "request": "GET /french/news/3004tick.htm HTTP/1.0", "status": 200, "size": 4234} +{"@timestamp": 893966223, "clientip":"47.135.0.0", "request": "GET /french/images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966224, "clientip":"47.135.0.0", "request": "GET /images/ps_bdr_r.gif HTTP/1.0", "status": 200, "size": 281} +{"@timestamp": 893966224, "clientip":"47.135.0.0", "request": "GET /images/ps_bdr_l.gif HTTP/1.0", "status": 200, "size": 346} +{"@timestamp": 893966232, "clientip":"248.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966259, "clientip":"2.2.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966273, "clientip":"218.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966280, "clientip":"48.4.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966296, "clientip":"128.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966306, "clientip":"139.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966310, "clientip":"141.78.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966310, "clientip":"141.78.0.0", "request": "GET /french/nav_top_inet.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966310, "clientip":"141.78.0.0", "request": "GET /french/nav_inet.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966311, "clientip":"141.78.0.0", "request": "GET /french/splash_inet.html HTTP/1.0", "status": 200, "size": 3781} +{"@timestamp": 893966322, "clientip":"249.1.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966327, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893966342, "clientip":"47.135.0.0", "request": "GET /french/frntpage.htm HTTP/1.0", "status": 200, "size": 12824} +{"@timestamp": 893966343, "clientip":"141.78.0.0", "request": "GET /french/ProScroll.class HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966344, "clientip":"141.78.0.0", "request": "GET /french/images/nav_history_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966344, "clientip":"141.78.0.0", "request": "GET /french/images/nav_store_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966344, "clientip":"141.78.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893966344, "clientip":"141.78.0.0", "request": "GET /french/images/nav_tickets_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966345, "clientip":"141.78.0.0", "request": "GET /images/hm_brdr.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966345, "clientip":"141.78.0.0", "request": "GET /images/hm_anime_f.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966345, "clientip":"141.78.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893966345, "clientip":"141.78.0.0", "request": "GET /images/hm_day_f.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966346, "clientip":"141.78.0.0", "request": "GET /french/images/nav_team_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966346, "clientip":"141.78.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966346, "clientip":"141.78.0.0", "request": "GET /french/images/nav_history_off.gif HTTP/1.0", "status": 206, "size": 838} +{"@timestamp": 893966347, "clientip":"141.78.0.0", "request": "GET /images/hm_brdr.gif HTTP/1.0", "status": 206, "size": 107} +{"@timestamp": 893966348, "clientip":"141.78.0.0", "request": "GET /french/images/nav_field_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966348, "clientip":"141.78.0.0", "request": "GET /french/images/nav_sitemap_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966348, "clientip":"141.78.0.0", "request": "GET /french/images/nav_home_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966348, "clientip":"141.78.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 200, "size": 208} +{"@timestamp": 893966348, "clientip":"141.78.0.0", "request": "GET /images/dot.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966349, "clientip":"141.78.0.0", "request": "GET /french/images/nav_venue_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966350, "clientip":"141.78.0.0", "request": "GET /french/images/nav_hosts_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966351, "clientip":"141.78.0.0", "request": "GET /images/hm_linkf.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966351, "clientip":"141.78.0.0", "request": "GET /french/images/nav_logo_sponsors.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966352, "clientip":"141.78.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966352, "clientip":"141.78.0.0", "request": "GET /images/info.gif HTTP/1.0", "status": 200, "size": 1251} +{"@timestamp": 893966368, "clientip":"221.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966381, "clientip":"43.135.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966384, "clientip":"175.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966384, "clientip":"55.0.0.0", "request": "GET /french/index.html HTTP/1.0", "status": 200, "size": 985} +{"@timestamp": 893966386, "clientip":"141.78.0.0", "request": "GET /french/competition/maincomp.htm HTTP/1.0", "status": 200, "size": 3048} +{"@timestamp": 893966388, "clientip":"141.78.0.0", "request": "GET /images/comp_hm_brac.gif HTTP/1.0", "status": 200, "size": 254} +{"@timestamp": 893966388, "clientip":"141.78.0.0", "request": "GET /french/images/comp_bg2_hm.jpg HTTP/1.0", "status": 200, "size": 25676} +{"@timestamp": 893966388, "clientip":"141.78.0.0", "request": "GET /french/images/bar.jpg HTTP/1.0", "status": 200, "size": 686} +{"@timestamp": 893966388, "clientip":"141.78.0.0", "request": "GET /french/images/comp_hm_header_shad.gif HTTP/1.0", "status": 200, "size": 2207} +{"@timestamp": 893966388, "clientip":"141.78.0.0", "request": "GET /french/images/france98b.gif HTTP/1.0", "status": 200, "size": 2122} +{"@timestamp": 893966389, "clientip":"141.78.0.0", "request": "GET /french/images/comp_hm_nav.gif HTTP/1.0", "status": 200, "size": 11652} +{"@timestamp": 893966390, "clientip":"141.78.0.0", "request": "GET /images/comp_hm_archive.gif HTTP/1.0", "status": 200, "size": 1644} +{"@timestamp": 893966391, "clientip":"141.78.0.0", "request": "GET /french/frntpage.htm HTTP/1.0", "status": 200, "size": 12824} +{"@timestamp": 893966392, "clientip":"141.78.0.0", "request": "GET /images/backnews.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966392, "clientip":"141.78.0.0", "request": "GET /french/images/fpnewstop.gif HTTP/1.0", "status": 200, "size": 1317} +{"@timestamp": 893966392, "clientip":"141.78.0.0", "request": "GET /french/images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966393, "clientip":"141.78.0.0", "request": "GET /french/images/news_btn_letter_off.gif HTTP/1.0", "status": 200, "size": 871} +{"@timestamp": 893966393, "clientip":"141.78.0.0", "request": "GET /images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966393, "clientip":"141.78.0.0", "request": "GET /images/ligneb.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966393, "clientip":"141.78.0.0", "request": "GET /french/images/today_new.gif HTTP/1.0", "status": 200, "size": 869} +{"@timestamp": 893966394, "clientip":"141.78.0.0", "request": "GET /images/case5.gif HTTP/1.0", "status": 200, "size": 1362} +{"@timestamp": 893966394, "clientip":"141.78.0.0", "request": "GET /images/ligne.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966395, "clientip":"141.78.0.0", "request": "GET /images/ligneb.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966395, "clientip":"141.78.0.0", "request": "GET /images/ligne01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966395, "clientip":"141.78.0.0", "request": "GET /images/ligneb.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966395, "clientip":"141.78.0.0", "request": "GET /french/images/news_btn_press_off.gif HTTP/1.0", "status": 200, "size": 1795} +{"@timestamp": 893966396, "clientip":"141.78.0.0", "request": "GET /images/ligne01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966397, "clientip":"141.78.0.0", "request": "GET /images/bord_stories01.gif HTTP/1.0", "status": 200, "size": 333} +{"@timestamp": 893966397, "clientip":"141.78.0.0", "request": "GET /images/dburton.jpg HTTP/1.0", "status": 200, "size": 12009} +{"@timestamp": 893966397, "clientip":"141.78.0.0", "request": "GET /images/ligne1_case5.gif HTTP/1.0", "status": 200, "size": 1018} +{"@timestamp": 893966397, "clientip":"141.78.0.0", "request": "GET /images/ligne4_latebreak.gif HTTP/1.0", "status": 200, "size": 1056} +{"@timestamp": 893966397, "clientip":"42.135.0.0", "request": "GET /fth.htm HTTP/1.1", "status": 200, "size": 190} +{"@timestamp": 893966398, "clientip":"141.78.0.0", "request": "GET /french/images/news_btn_kits_off.gif HTTP/1.0", "status": 200, "size": 965} +{"@timestamp": 893966398, "clientip":"141.78.0.0", "request": "GET /images/ligneb01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966398, "clientip":"141.78.0.0", "request": "GET /images/bord_stories.gif HTTP/1.0", "status": 200, "size": 520} +{"@timestamp": 893966399, "clientip":"141.78.0.0", "request": "GET /images/base.gif HTTP/1.0", "status": 200, "size": 366} +{"@timestamp": 893966399, "clientip":"141.78.0.0", "request": "GET /french/images/lateb_new.gif HTTP/1.0", "status": 200, "size": 1285} +{"@timestamp": 893966399, "clientip":"141.78.0.0", "request": "GET /images/ligneb01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966399, "clientip":"141.78.0.0", "request": "GET /french/images/archives.gif HTTP/1.0", "status": 200, "size": 569} +{"@timestamp": 893966400, "clientip":"141.78.0.0", "request": "GET /french/images/top_stories.gif HTTP/1.0", "status": 200, "size": 1078} +{"@timestamp": 893966402, "clientip":"0.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.0", "status": 200, "size": 60349} +{"@timestamp": 893966403, "clientip":"1.0.0.0", "request": "GET /images/home_bg_stars.gif HTTP/1.0", "status": 200, "size": 2557} +{"@timestamp": 893966403, "clientip":"1.0.0.0", "request": "GET /images/home_fr_phrase.gif HTTP/1.0", "status": 200, "size": 2843} +{"@timestamp": 893966403, "clientip":"2.0.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893966403, "clientip":"1.0.0.0", "request": "GET /images/home_logo.gif HTTP/1.0", "status": 200, "size": 3401} +{"@timestamp": 893966404, "clientip":"2.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893966404, "clientip":"1.0.0.0", "request": "GET /images/home_eng_phrase.gif HTTP/1.0", "status": 200, "size": 2861} +{"@timestamp": 893966404, "clientip":"3.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966404, "clientip":"4.0.0.0", "request": "GET /english/frntpage.htm HTTP/1.0", "status": 200, "size": 12800} +{"@timestamp": 893966404, "clientip":"5.0.0.0", "request": "GET /images/hm_f98_top.gif HTTP/1.1", "status": 200, "size": 915} +{"@timestamp": 893966404, "clientip":"6.0.0.0", "request": "GET /images/team_hm_concacaf.gif HTTP/1.0", "status": 200, "size": 764} +{"@timestamp": 893966404, "clientip":"6.0.0.0", "request": "GET /images/team_hm_afc.gif HTTP/1.0", "status": 200, "size": 475} +{"@timestamp": 893966404, "clientip":"6.0.0.0", "request": "GET /images/team_hm_caf.gif HTTP/1.0", "status": 200, "size": 473} +{"@timestamp": 893966405, "clientip":"7.0.0.0", "request": "GET /english/playing/mascot/mascot.html HTTP/1.0", "status": 200, "size": 5521} +{"@timestamp": 893966405, "clientip":"1.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.0", "status": 200, "size": 327} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /english/images/comp_bu_stage1n.gif HTTP/1.0", "status": 200, "size": 1548} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /english/images/comp_bu_stage2n_on.gif HTTP/1.0", "status": 200, "size": 996} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc_top.gif HTTP/1.0", "status": 200, "size": 163} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc_topr.gif HTTP/1.0", "status": 200, "size": 163} +{"@timestamp": 893966405, "clientip":"9.0.0.0", "request": "GET /english/history/past_cups/images/posters/france38.gif HTTP/1.0", "status": 200, "size": 4649} +{"@timestamp": 893966405, "clientip":"3.0.0.0", "request": "GET /english/nav_top_inet.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966405, "clientip":"3.0.0.0", "request": "GET /english/nav_inet.html HTTP/1.0", "status": 200, "size": 2672} +{"@timestamp": 893966405, "clientip":"3.0.0.0", "request": "GET /english/splash_inet.html HTTP/1.0", "status": 200, "size": 3730} +{"@timestamp": 893966405, "clientip":"9.0.0.0", "request": "GET /english/history/past_cups/images/38-1.jpg HTTP/1.0", "status": 200, "size": 14315} +{"@timestamp": 893966405, "clientip":"10.0.0.0", "request": "GET /images/cal_nant.gif HTTP/1.0", "status": 200, "size": 359} +{"@timestamp": 893966405, "clientip":"9.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_38_on.gif HTTP/1.0", "status": 200, "size": 507} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc_botr.gif HTTP/1.0", "status": 200, "size": 158} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc_bot.gif HTTP/1.0", "status": 200, "size": 160} +{"@timestamp": 893966405, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc.gif HTTP/1.0", "status": 200, "size": 82} +{"@timestamp": 893966405, "clientip":"11.0.0.0", "request": "GET /images/comp_stage2_brc_botr.gif HTTP/1.0", "status": 200, "size": 158} +{"@timestamp": 893966405, "clientip":"12.0.0.0", "request": "GET /english/nav_top_inet.html HTTP/1.0", "status": 200, "size": 374} +{"@timestamp": 893966405, "clientip":"10.0.0.0", "request": "GET /images/cal_lyon.gif HTTP/1.0", "status": 200, "size": 286} +{"@timestamp": 893966405, "clientip":"12.0.0.0", "request": "GET /english/nav_inet.html HTTP/1.0", "status": 200, "size": 2672} +{"@timestamp": 893966405, "clientip":"1.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.0", "status": 200, "size": 2491} +{"@timestamp": 893966405, "clientip":"12.0.0.0", "request": "GET /english/splash_inet.html HTTP/1.0", "status": 200, "size": 3730} +{"@timestamp": 893966405, "clientip":"10.0.0.0", "request": "GET /images/cal_mars.gif HTTP/1.0", "status": 200, "size": 377} +{"@timestamp": 893966405, "clientip":"10.0.0.0", "request": "GET /images/cal-lens.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966405, "clientip":"10.0.0.0", "request": "GET /images/cal_toul.gif HTTP/1.0", "status": 200, "size": 380} +{"@timestamp": 893966406, "clientip":"8.0.0.0", "request": "GET /english/competition/stage2.htm HTTP/1.0", "status": 200, "size": 16606} +{"@timestamp": 893966406, "clientip":"12.0.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 200, "size": 33665} +{"@timestamp": 893966406, "clientip":"10.0.0.0", "request": "GET /images/cal-lens.gif HTTP/1.0", "status": 206, "size": 124} +{"@timestamp": 893966406, "clientip":"8.0.0.0", "request": "GET /english/images/comp_bg2_hm.jpg HTTP/1.0", "status": 200, "size": 25676} +{"@timestamp": 893966406, "clientip":"8.0.0.0", "request": "GET /english/images/comp_hm_header.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"10.0.0.0", "request": "GET /images/cal_mont.gif HTTP/1.0", "status": 200, "size": 316} +{"@timestamp": 893966406, "clientip":"8.0.0.0", "request": "GET /english/images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"2.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 934} +{"@timestamp": 893966406, "clientip":"10.0.0.0", "request": "GET /images/cal_steti.gif HTTP/1.0", "status": 200, "size": 1125} +{"@timestamp": 893966406, "clientip":"1.0.0.0", "request": "GET /images/home_eng_button.gif HTTP/1.0", "status": 200, "size": 1927} +{"@timestamp": 893966406, "clientip":"5.0.0.0", "request": "GET /french/images/news_btn_press_off.gif HTTP/1.1", "status": 200, "size": 1795} +{"@timestamp": 893966406, "clientip":"1.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.0", "status": 200, "size": 60349} +{"@timestamp": 893966406, "clientip":"13.0.0.0", "request": "GET /english/images/comp_hm_header.gif HTTP/1.0", "status": 200, "size": 1189} +{"@timestamp": 893966406, "clientip":"3.0.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"14.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.1", "status": 200, "size": 1504} +{"@timestamp": 893966406, "clientip":"3.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"15.0.0.0", "request": "GET /cgi-bin/trivia/Trivia.pl?ENG HTTP/1.0", "status": 200, "size": 6213} +{"@timestamp": 893966406, "clientip":"13.0.0.0", "request": "GET /english/images/france98b.gif HTTP/1.0", "status": 200, "size": 2122} +{"@timestamp": 893966406, "clientip":"13.0.0.0", "request": "GET /english/images/comp_bu_stage1n.gif HTTP/1.0", "status": 200, "size": 1548} +{"@timestamp": 893966406, "clientip":"16.0.0.0", "request": "GET /images/s102336.gif HTTP/1.0", "status": 200, "size": 177} +{"@timestamp": 893966406, "clientip":"17.0.0.0", "request": "GET /english/history/images/france98b.GIF HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966406, "clientip":"17.0.0.0", "request": "GET /english/history/images/football.GIF HTTP/1.0", "status": 200, "size": 1170} +{"@timestamp": 893966406, "clientip":"18.0.0.0", "request": "GET /english/competition/headtohead78.htm HTTP/1.0", "status": 200, "size": 20126} +{"@timestamp": 893966406, "clientip":"3.0.0.0", "request": "GET /english/images/nav_tickets_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"12.0.0.0", "request": "GET /images/hm_anime_e.gif HTTP/1.0", "status": 200, "size": 15609} +{"@timestamp": 893966406, "clientip":"17.0.0.0", "request": "GET /english/history/images/infrance.GIF HTTP/1.0", "status": 200, "size": 990} +{"@timestamp": 893966406, "clientip":"19.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.0", "status": 200, "size": 853} +{"@timestamp": 893966406, "clientip":"141.78.0.0", "request": "GET /french/images/hm_f98_top.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"141.78.0.0", "request": "GET /images/ps_bdr_r.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966406, "clientip":"141.78.0.0", "request": "GET /french/news/3004tick.htm HTTP/1.0", "status": 200, "size": 4234} +{"@timestamp": 893966406, "clientip":"141.78.0.0", "request": "GET /images/ps_bdr_l.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966407, "clientip":"12.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893966407, "clientip":"19.0.0.0", "request": "GET /english/images/nav_comp_off.gif HTTP/1.0", "status": 200, "size": 994} +{"@timestamp": 893966407, "clientip":"3.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893966407, "clientip":"19.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893966407, "clientip":"8.0.0.0", "request": "GET /english/images/france98b.gif HTTP/1.0", "status": 200, "size": 2122} +{"@timestamp": 893966407, "clientip":"3.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966407, "clientip":"3.0.0.0", "request": "GET /english/images/nav_comp_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966407, "clientip":"1.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.0", "status": 200, "size": 2140} +{"@timestamp": 893966407, "clientip":"12.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966407, "clientip":"3.0.0.0", "request": "GET /english/images/nav_team_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966407, "clientip":"3.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966407, "clientip":"12.0.0.0", "request": "GET /english/images/hm_official.gif HTTP/1.0", "status": 200, "size": 1807} +{"@timestamp": 893966407, "clientip":"12.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 200, "size": 6507} +{"@timestamp": 893966407, "clientip":"20.0.0.0", "request": "GET /images/10982.gif HTTP/1.0", "status": 200, "size": 183} +{"@timestamp": 893966407, "clientip":"20.0.0.0", "request": "GET /images/11101.gif HTTP/1.0", "status": 200, "size": 415} +{"@timestamp": 893966407, "clientip":"0.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.0", "status": 200, "size": 2491} +{"@timestamp": 893966407, "clientip":"5.0.0.0", "request": "GET /french/images/space.gif HTTP/1.1", "status": 200, "size": 42} +{"@timestamp": 893966407, "clientip":"5.0.0.0", "request": "GET /french/images/fpnewstop.gif HTTP/1.1", "status": 200, "size": 1317} +{"@timestamp": 893966407, "clientip":"13.0.0.0", "request": "GET /english/images/comp_bu_stage2n.gif HTTP/1.0", "status": 200, "size": 984} +{"@timestamp": 893966407, "clientip":"2.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966407, "clientip":"13.0.0.0", "request": "GET /english/images/comp_bu_groupsn_on.gif HTTP/1.0", "status": 200, "size": 963} +{"@timestamp": 893966407, "clientip":"20.0.0.0", "request": "GET /images/bordeaux.gif HTTP/1.0", "status": 200, "size": 723} +{"@timestamp": 893966407, "clientip":"10.0.0.0", "request": "GET /images/cal_bord.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966407, "clientip":"2.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.0", "status": 200, "size": 870} +{"@timestamp": 893966407, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/banner.jpg HTTP/1.0", "status": 200, "size": 19967} +{"@timestamp": 893966407, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/01test.gif HTTP/1.0", "status": 200, "size": 1664} +{"@timestamp": 893966407, "clientip":"8.0.0.0", "request": "GET /english/images/comp_bu_stage2n_on.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966408, "clientip":"8.0.0.0", "request": "GET /english/images/comp_bu_refsn.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966408, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/quizbg.gif HTTP/1.0", "status": 200, "size": 1460} +{"@timestamp": 893966408, "clientip":"20.0.0.0", "request": "GET /images/lyon.gif HTTP/1.0", "status": 200, "size": 599} +{"@timestamp": 893966408, "clientip":"12.0.0.0", "request": "GET /images/hm_day_e.gif HTTP/1.0", "status": 200, "size": 499} +{"@timestamp": 893966408, "clientip":"12.0.0.0", "request": "GET /images/info.gif HTTP/1.0", "status": 200, "size": 1251} +{"@timestamp": 893966408, "clientip":"12.0.0.0", "request": "GET /english/images/nav_team_off.gif HTTP/1.0", "status": 200, "size": 776} +{"@timestamp": 893966408, "clientip":"2.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966408, "clientip":"12.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893966408, "clientip":"2.0.0.0", "request": "GET /english/images/hm_official.gif HTTP/1.0", "status": 200, "size": 1807} +{"@timestamp": 893966408, "clientip":"12.0.0.0", "request": "GET /english/images/nav_comp_off.gif HTTP/1.0", "status": 200, "size": 994} +{"@timestamp": 893966408, "clientip":"21.0.0.0", "request": "GET /english/news/11415.htm HTTP/1.1", "status": 200, "size": 21300} +{"@timestamp": 893966408, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc_top.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966408, "clientip":"12.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 1005} +{"@timestamp": 893966408, "clientip":"17.0.0.0", "request": "GET /english/history/images/reading.GIF HTTP/1.0", "status": 200, "size": 1171} +{"@timestamp": 893966408, "clientip":"22.0.0.0", "request": "GET /english/teams/teamqualify124.htm HTTP/1.0", "status": 200, "size": 3866} +{"@timestamp": 893966408, "clientip":"23.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966408, "clientip":"24.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966409, "clientip":"24.0.0.0", "request": "GET /english/playing/images/anim/trivia_on.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966409, "clientip":"5.0.0.0", "request": "GET /french/images/today_new.gif HTTP/1.1", "status": 200, "size": 869} +{"@timestamp": 893966409, "clientip":"20.0.0.0", "request": "GET /images/saintdenis.gif HTTP/1.0", "status": 200, "size": 702} +{"@timestamp": 893966409, "clientip":"24.0.0.0", "request": "GET /english/playing/images/play_hm_mascot.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966409, "clientip":"5.0.0.0", "request": "GET /french/images/top_stories.gif HTTP/1.1", "status": 200, "size": 1078} +{"@timestamp": 893966409, "clientip":"5.0.0.0", "request": "GET /images/case5.gif HTTP/1.1", "status": 200, "size": 1362} +{"@timestamp": 893966409, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_30_off.gif HTTP/1.0", "status": 200, "size": 406} +{"@timestamp": 893966409, "clientip":"20.0.0.0", "request": "GET /images/toulouse.gif HTTP/1.0", "status": 200, "size": 704} +{"@timestamp": 893966409, "clientip":"25.0.0.0", "request": "GET /images/home_bg_stars.gif HTTP/1.1", "status": 200, "size": 2557} +{"@timestamp": 893966409, "clientip":"26.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 1005} +{"@timestamp": 893966409, "clientip":"27.0.0.0", "request": "GET /english/images/teams_bu_group_on.gif HTTP/1.0", "status": 200, "size": 668} +{"@timestamp": 893966409, "clientip":"13.0.0.0", "request": "GET /english/images/comp_bu_refsn.gif HTTP/1.0", "status": 200, "size": 933} +{"@timestamp": 893966409, "clientip":"28.0.0.0", "request": "GET /english/individuals/player13893.htm HTTP/1.0", "status": 200, "size": 6472} +{"@timestamp": 893966409, "clientip":"13.0.0.0", "request": "GET /english/images/comp_bu_calendar.gif HTTP/1.0", "status": 200, "size": 1197} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_fr_phrase.gif HTTP/1.0", "status": 200, "size": 2843} +{"@timestamp": 893966410, "clientip":"12.0.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_logo.gif HTTP/1.0", "status": 200, "size": 3401} +{"@timestamp": 893966410, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/02how.gif HTTP/1.0", "status": 200, "size": 754} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_eng_phrase.gif HTTP/1.0", "status": 200, "size": 2861} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.0", "status": 200, "size": 327} +{"@timestamp": 893966410, "clientip":"29.0.0.0", "request": "GET /images/10511.jpg HTTP/1.0", "status": 200, "size": 15543} +{"@timestamp": 893966410, "clientip":"20.0.0.0", "request": "GET /images/11289.jpg HTTP/1.0", "status": 200, "size": 6444} +{"@timestamp": 893966410, "clientip":"12.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966410, "clientip":"12.0.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 200, "size": 208} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.0", "status": 200, "size": 2491} +{"@timestamp": 893966410, "clientip":"20.0.0.0", "request": "GET /images/11288.jpg HTTP/1.0", "status": 200, "size": 5874} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_eng_button.gif HTTP/1.0", "status": 200, "size": 1927} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.0", "status": 200, "size": 2140} +{"@timestamp": 893966410, "clientip":"8.0.0.0", "request": "GET /images/comp_stage2_brc.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966410, "clientip":"5.0.0.0", "request": "GET /images/dburton.jpg HTTP/1.1", "status": 200, "size": 12009} +{"@timestamp": 893966410, "clientip":"2.0.0.0", "request": "GET /images/hm_anime_e.gif HTTP/1.0", "status": 200, "size": 15609} +{"@timestamp": 893966410, "clientip":"12.0.0.0", "request": "GET /images/hm_brdr.gif HTTP/1.0", "status": 200, "size": 235} +{"@timestamp": 893966410, "clientip":"12.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.0", "status": 200, "size": 853} +{"@timestamp": 893966410, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/04luck.gif HTTP/1.0", "status": 200, "size": 744} +{"@timestamp": 893966410, "clientip":"23.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.0", "status": 200, "size": 60349} +{"@timestamp": 893966410, "clientip":"2.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 200, "size": 6507} +{"@timestamp": 893966410, "clientip":"5.0.0.0", "request": "GET /images/bord_stories01.gif HTTP/1.1", "status": 200, "size": 333} +{"@timestamp": 893966410, "clientip":"12.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.0", "status": 200, "size": 870} +{"@timestamp": 893966411, "clientip":"30.0.0.0", "request": "GET /english/playing/download/download.html HTTP/1.0", "status": 200, "size": 13898} +{"@timestamp": 893966411, "clientip":"2.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966411, "clientip":"16.0.0.0", "request": "GET /images/s102373.gif HTTP/1.0", "status": 200, "size": 142} +{"@timestamp": 893966411, "clientip":"3.0.0.0", "request": "GET /images/hm_day_e.gif HTTP/1.0", "status": 200, "size": 499} +{"@timestamp": 893966411, "clientip":"3.0.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 200, "size": 33665} +{"@timestamp": 893966411, "clientip":"12.0.0.0", "request": "GET /english/images/nav_tickets_off.gif HTTP/1.0", "status": 200, "size": 937} +{"@timestamp": 893966411, "clientip":"3.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966411, "clientip":"3.0.0.0", "request": "GET /english/images/hm_official.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966411, "clientip":"5.0.0.0", "request": "GET /images/bord_stories.gif HTTP/1.1", "status": 200, "size": 520} +{"@timestamp": 893966411, "clientip":"3.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966411, "clientip":"31.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.1", "status": 200, "size": 934} +{"@timestamp": 893966411, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/03score.gif HTTP/1.0", "status": 200, "size": 572} +{"@timestamp": 893966411, "clientip":"3.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966411, "clientip":"31.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.1", "status": 200, "size": 914} +{"@timestamp": 893966411, "clientip":"32.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.1", "status": 200, "size": 60349} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /images/hm_linkf.gif HTTP/1.0", "status": 200, "size": 123} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966412, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia/submit.gif HTTP/1.0", "status": 200, "size": 1808} +{"@timestamp": 893966412, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_header.gif HTTP/1.0", "status": 200, "size": 1989} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 934} +{"@timestamp": 893966412, "clientip":"23.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 200, "size": 892} +{"@timestamp": 893966412, "clientip":"12.0.0.0", "request": "GET /images/dot.gif HTTP/1.0", "status": 200, "size": 43} +{"@timestamp": 893966412, "clientip":"5.0.0.0", "request": "GET /french/images/lateb_new.gif HTTP/1.1", "status": 200, "size": 1285} +{"@timestamp": 893966412, "clientip":"19.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 1005} +{"@timestamp": 893966412, "clientip":"25.0.0.0", "request": "GET /images/home_eng_phrase.gif HTTP/1.1", "status": 200, "size": 2861} +{"@timestamp": 893966412, "clientip":"19.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.0", "status": 200, "size": 870} +{"@timestamp": 893966412, "clientip":"23.0.0.0", "request": "GET /english/nav_top_inet.html HTTP/1.0", "status": 200, "size": 374} +{"@timestamp": 893966412, "clientip":"23.0.0.0", "request": "GET /english/nav_inet.html HTTP/1.0", "status": 200, "size": 2672} +{"@timestamp": 893966412, "clientip":"33.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.1", "status": 200, "size": 2140} +{"@timestamp": 893966413, "clientip":"25.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.1", "status": 200, "size": 2140} +{"@timestamp": 893966413, "clientip":"23.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893966413, "clientip":"17.0.0.0", "request": "GET /english/history/images/thecup.GIF HTTP/1.0", "status": 200, "size": 1293} +{"@timestamp": 893966413, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_34_off.gif HTTP/1.0", "status": 200, "size": 397} +{"@timestamp": 893966413, "clientip":"23.0.0.0", "request": "GET /english/images/nav_team_off.gif HTTP/1.0", "status": 200, "size": 776} +{"@timestamp": 893966413, "clientip":"12.0.0.0", "request": "GET /english/images/nav_home_off.gif HTTP/1.0", "status": 200, "size": 828} +{"@timestamp": 893966413, "clientip":"16.0.0.0", "request": "GET /images/s102338.gif HTTP/1.0", "status": 200, "size": 138} +{"@timestamp": 893966413, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_38_off.gif HTTP/1.0", "status": 200, "size": 436} +{"@timestamp": 893966413, "clientip":"3.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966413, "clientip":"23.0.0.0", "request": "GET /english/images/nav_comp_off.gif HTTP/1.0", "status": 200, "size": 994} +{"@timestamp": 893966413, "clientip":"20.0.0.0", "request": "GET /images/11103.gif HTTP/1.0", "status": 200, "size": 513} +{"@timestamp": 893966413, "clientip":"3.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966413, "clientip":"23.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 200, "size": 6507} +{"@timestamp": 893966413, "clientip":"3.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966413, "clientip":"23.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 934} +{"@timestamp": 893966413, "clientip":"3.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966413, "clientip":"34.0.0.0", "request": "GET /english/playing/body.html HTTP/1.0", "status": 200, "size": 5033} +{"@timestamp": 893966413, "clientip":"19.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966413, "clientip":"3.0.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 200, "size": 208} +{"@timestamp": 893966414, "clientip":"35.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966414, "clientip":"36.0.0.0", "request": "GET /english/frntpage.htm HTTP/1.0", "status": 200, "size": 12800} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.0", "status": 200, "size": 870} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966414, "clientip":"20.0.0.0", "request": "GET /images/marseille.gif HTTP/1.0", "status": 200, "size": 722} +{"@timestamp": 893966414, "clientip":"5.0.0.0", "request": "GET /images/ligne4_latebreak.gif HTTP/1.1", "status": 200, "size": 1056} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /images/hm_linkf.gif HTTP/1.0", "status": 200, "size": 123} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 1005} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /english/images/nav_tickets_off.gif HTTP/1.0", "status": 200, "size": 937} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966414, "clientip":"24.0.0.0", "request": "GET /cgi-bin/trivia/Trivia.pl?ENG HTTP/1.0", "status": 200, "size": 6228} +{"@timestamp": 893966414, "clientip":"5.0.0.0", "request": "GET /images/ligneb01.gif HTTP/1.1", "status": 200, "size": 169} +{"@timestamp": 893966414, "clientip":"12.0.0.0", "request": "GET /english/tickets/body.html HTTP/1.0", "status": 200, "size": 2925} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966414, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_50_on.gif HTTP/1.0", "status": 200, "size": 455} +{"@timestamp": 893966414, "clientip":"23.0.0.0", "request": "GET /images/hm_anime_e.gif HTTP/1.0", "status": 200, "size": 15609} +{"@timestamp": 893966414, "clientip":"3.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966414, "clientip":"37.0.0.0", "request": "GET /french/competition/schedule.htm HTTP/1.0", "status": 200, "size": 49256} +{"@timestamp": 893966414, "clientip":"3.0.0.0", "request": "GET /images/hm_brdr.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966414, "clientip":"26.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966414, "clientip":"232.41.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8749} +{"@timestamp": 893966415, "clientip":"232.41.0.0", "request": "GET /images/home_tool.gif HTTP/1.0", "status": 200, "size": 327} +{"@timestamp": 893966415, "clientip":"19.0.0.0", "request": "GET /english/images/nav_home_off.gif HTTP/1.0", "status": 200, "size": 828} +{"@timestamp": 893966415, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_58_off.gif HTTP/1.0", "status": 200, "size": 397} +{"@timestamp": 893966415, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_hm_bg.jpg HTTP/1.0", "status": 200, "size": 20929} +{"@timestamp": 893966415, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_hm_header.gif HTTP/1.0", "status": 200, "size": 1226} +{"@timestamp": 893966415, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_hm_nav.gif HTTP/1.0", "status": 200, "size": 11253} +{"@timestamp": 893966415, "clientip":"12.0.0.0", "request": "GET /images/arw_red_lk.gif HTTP/1.0", "status": 200, "size": 1068} +{"@timestamp": 893966415, "clientip":"23.0.0.0", "request": "GET /english/images/nav_home_off.gif HTTP/1.0", "status": 200, "size": 828} +{"@timestamp": 893966415, "clientip":"23.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966415, "clientip":"21.0.0.0", "request": "GET /english/images/news_hm_header.gif HTTP/1.1", "status": 200, "size": 527} +{"@timestamp": 893966415, "clientip":"21.0.0.0", "request": "GET /images/bg_generic.jpg HTTP/1.1", "status": 200, "size": 21127} +{"@timestamp": 893966415, "clientip":"38.0.0.0", "request": "GET /english/teams/teamgroup.htm HTTP/1.0", "status": 200, "size": 11971} +{"@timestamp": 893966415, "clientip":"21.0.0.0", "request": "GET /english/images/space.gif HTTP/1.1", "status": 200, "size": 42} +{"@timestamp": 893966415, "clientip":"23.0.0.0", "request": "GET /images/info.gif HTTP/1.0", "status": 200, "size": 1251} +{"@timestamp": 893966415, "clientip":"34.0.0.0", "request": "GET /english/playing/images/play_hm_bg_opt1.jpg HTTP/1.0", "status": 200, "size": 44502} +{"@timestamp": 893966415, "clientip":"5.0.0.0", "request": "GET /images/ligne.gif HTTP/1.1", "status": 200, "size": 169} +{"@timestamp": 893966415, "clientip":"3.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966415, "clientip":"12.0.0.0", "request": "GET /images/arw_lk.gif HTTP/1.0", "status": 200, "size": 669} +{"@timestamp": 893966415, "clientip":"19.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966415, "clientip":"35.0.0.0", "request": "GET /images/home_bg_stars.gif HTTP/1.0", "status": 200, "size": 2557} +{"@timestamp": 893966416, "clientip":"23.0.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 200, "size": 208} +{"@timestamp": 893966416, "clientip":"5.0.0.0", "request": "GET /french/images/news_btn_kits_off.gif HTTP/1.1", "status": 200, "size": 965} +{"@timestamp": 893966416, "clientip":"23.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966416, "clientip":"23.0.0.0", "request": "GET /english/images/hm_official.gif HTTP/1.0", "status": 200, "size": 1807} +{"@timestamp": 893966416, "clientip":"23.0.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966416, "clientip":"19.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966416, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_62_off.gif HTTP/1.0", "status": 200, "size": 437} +{"@timestamp": 893966416, "clientip":"23.0.0.0", "request": "GET /images/hm_day_e.gif HTTP/1.0", "status": 200, "size": 499} +{"@timestamp": 893966416, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_66_off.gif HTTP/1.0", "status": 200, "size": 435} +{"@timestamp": 893966416, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_70_off.gif HTTP/1.0", "status": 200, "size": 409} +{"@timestamp": 893966416, "clientip":"39.0.0.0", "request": "GET /english/history/images/history_hm_header.gif HTTP/1.1", "status": 200, "size": 688} +{"@timestamp": 893966416, "clientip":"39.0.0.0", "request": "GET /english/history/images/history_hm_posters.jpg HTTP/1.1", "status": 200, "size": 33296} +{"@timestamp": 893966416, "clientip":"201.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966417, "clientip":"25.0.0.0", "request": "GET /images/home_eng_button.gif HTTP/1.1", "status": 200, "size": 1927} +{"@timestamp": 893966417, "clientip":"23.0.0.0", "request": "GET /images/dot.gif HTTP/1.0", "status": 200, "size": 43} +{"@timestamp": 893966417, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_74_off.gif HTTP/1.0", "status": 200, "size": 423} +{"@timestamp": 893966417, "clientip":"1.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 200, "size": 892} +{"@timestamp": 893966417, "clientip":"5.0.0.0", "request": "GET /images/ligneb.gif HTTP/1.1", "status": 200, "size": 169} +{"@timestamp": 893966417, "clientip":"35.0.0.0", "request": "GET /images/home_eng_phrase.gif HTTP/1.0", "status": 200, "size": 2861} +{"@timestamp": 893966417, "clientip":"35.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.0", "status": 200, "size": 2140} +{"@timestamp": 893966417, "clientip":"35.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.0", "status": 200, "size": 327} +{"@timestamp": 893966417, "clientip":"35.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.0", "status": 200, "size": 2491} +{"@timestamp": 893966417, "clientip":"35.0.0.0", "request": "GET /images/home_eng_button.gif HTTP/1.0", "status": 200, "size": 1927} +{"@timestamp": 893966417, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_78_off.gif HTTP/1.0", "status": 200, "size": 427} +{"@timestamp": 893966417, "clientip":"16.0.0.0", "request": "GET /english/images/team_group_header_d.gif HTTP/1.0", "status": 200, "size": 646} +{"@timestamp": 893966418, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_82_off.gif HTTP/1.0", "status": 200, "size": 433} +{"@timestamp": 893966418, "clientip":"40.0.0.0", "request": "GET /english/history/past_cups/italy34.html HTTP/1.1", "status": 200, "size": 13022} +{"@timestamp": 893966418, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_86_off.gif HTTP/1.0", "status": 200, "size": 409} +{"@timestamp": 893966418, "clientip":"19.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 934} +{"@timestamp": 893966418, "clientip":"1.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 200, "size": 892} +{"@timestamp": 893966418, "clientip":"37.0.0.0", "request": "GET /french/images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966418, "clientip":"19.0.0.0", "request": "GET /english/images/nav_tickets_off.gif HTTP/1.0", "status": 200, "size": 937} +{"@timestamp": 893966418, "clientip":"19.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966418, "clientip":"8.0.0.0", "request": "GET /english/images/comp_bu_groupsn_on.gif HTTP/1.0", "status": 200, "size": 963} +{"@timestamp": 893966418, "clientip":"22.0.0.0", "request": "GET /english/images/team_bu_detail2.gif HTTP/1.0", "status": 200, "size": 1438} +{"@timestamp": 893966418, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_90_off.gif HTTP/1.0", "status": 200, "size": 429} +{"@timestamp": 893966418, "clientip":"38.0.0.0", "request": "GET /english/images/teams_bu_group_on.gif HTTP/1.0", "status": 200, "size": 668} +{"@timestamp": 893966419, "clientip":"20.0.0.0", "request": "GET /images/nantes.gif HTTP/1.0", "status": 200, "size": 677} +{"@timestamp": 893966419, "clientip":"16.0.0.0", "request": "GET /images/s102326.gif HTTP/1.0", "status": 200, "size": 248} +{"@timestamp": 893966419, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_94_off.gif HTTP/1.0", "status": 200, "size": 432} +{"@timestamp": 893966419, "clientip":"1.0.0.0", "request": "GET /english/nav_top_inet.html HTTP/1.0", "status": 200, "size": 374} +{"@timestamp": 893966419, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bracket_top.gif HTTP/1.0", "status": 200, "size": 289} +{"@timestamp": 893966419, "clientip":"20.0.0.0", "request": "GET /images/saintetienne.gif HTTP/1.0", "status": 200, "size": 761} +{"@timestamp": 893966419, "clientip":"20.0.0.0", "request": "GET /images/11287.jpg HTTP/1.0", "status": 200, "size": 6431} +{"@timestamp": 893966420, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/posters/brasil50.gif HTTP/1.0", "status": 200, "size": 5003} +{"@timestamp": 893966420, "clientip":"20.0.0.0", "request": "GET /images/lens.gif HTTP/1.0", "status": 200, "size": 582} +{"@timestamp": 893966420, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_bracket_bot.gif HTTP/1.0", "status": 200, "size": 631} +{"@timestamp": 893966420, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/past_trophy.gif HTTP/1.0", "status": 200, "size": 800} +{"@timestamp": 893966420, "clientip":"41.0.0.0", "request": "GET /images/col.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966421, "clientip":"17.0.0.0", "request": "GET /english/history/past_cups/images/50-1.jpg HTTP/1.0", "status": 200, "size": 15314} +{"@timestamp": 893966421, "clientip":"1.0.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966421, "clientip":"40.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_30_off.gif HTTP/1.1", "status": 200, "size": 406} +{"@timestamp": 893966421, "clientip":"40.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_34_on.gif HTTP/1.1", "status": 200, "size": 474} +{"@timestamp": 893966421, "clientip":"16.0.0.0", "request": "GET /images/s102477.gif HTTP/1.0", "status": 200, "size": 214} +{"@timestamp": 893966421, "clientip":"26.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966421, "clientip":"26.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 934} +{"@timestamp": 893966421, "clientip":"8.0.0.0", "request": "GET /english/teams/teamgroup.htm HTTP/1.0", "status": 200, "size": 11971} +{"@timestamp": 893966421, "clientip":"38.0.0.0", "request": "GET /images/s102325.gif HTTP/1.0", "status": 200, "size": 187} +{"@timestamp": 893966422, "clientip":"15.0.0.0", "request": "GET /english/playing/links.html HTTP/1.0", "status": 200, "size": 18694} +{"@timestamp": 893966422, "clientip":"40.0.0.0", "request": "GET /english/history/past_cups/images/34-1.jpg HTTP/1.1", "status": 200, "size": 14275} +{"@timestamp": 893966422, "clientip":"8.0.0.0", "request": "GET /images/teams_hm_bg.jpg HTTP/1.0", "status": 200, "size": 18794} +{"@timestamp": 893966422, "clientip":"42.0.0.0", "request": "GET /english/frntpage.htm HTTP/1.0", "status": 200, "size": 12800} +{"@timestamp": 893966422, "clientip":"25.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.1", "status": 200, "size": 2491} +{"@timestamp": 893966422, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_a.gif HTTP/1.0", "status": 200, "size": 639} +{"@timestamp": 893966422, "clientip":"8.0.0.0", "request": "GET /english/images/teams_bu_group_on.gif HTTP/1.0", "status": 200, "size": 668} +{"@timestamp": 893966422, "clientip":"8.0.0.0", "request": "GET /english/images/teams_bu_confed_off.gif HTTP/1.0", "status": 200, "size": 1105} +{"@timestamp": 893966422, "clientip":"43.0.0.0", "request": "GET /english/individuals/player111722.htm HTTP/1.0", "status": 200, "size": 6523} +{"@timestamp": 893966423, "clientip":"20.0.0.0", "request": "GET /images/montpellier.gif HTTP/1.0", "status": 200, "size": 728} +{"@timestamp": 893966423, "clientip":"12.0.0.0", "request": "GET /english/tickets/out_france.html HTTP/1.0", "status": 200, "size": 6143} +{"@timestamp": 893966423, "clientip":"20.0.0.0", "request": "GET /images/11105.gif HTTP/1.0", "status": 200, "size": 114} +{"@timestamp": 893966423, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_quest_bg2.jpg HTTP/1.0", "status": 200, "size": 11324} +{"@timestamp": 893966423, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_header.gif HTTP/1.0", "status": 200, "size": 453} +{"@timestamp": 893966423, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 1647} +{"@timestamp": 893966423, "clientip":"21.0.0.0", "request": "GET /english/images/news_bu_press_off.gif HTTP/1.1", "status": 200, "size": 847} +{"@timestamp": 893966423, "clientip":"44.0.0.0", "request": "GET /english/teams/teambio169.htm HTTP/1.1", "status": 200, "size": 11157} +{"@timestamp": 893966423, "clientip":"38.0.0.0", "request": "GET /images/s102327.gif HTTP/1.0", "status": 200, "size": 97} +{"@timestamp": 893966423, "clientip":"21.0.0.0", "request": "GET /english/images/news_bu_letter_off.gif HTTP/1.1", "status": 200, "size": 940} +{"@timestamp": 893966423, "clientip":"45.0.0.0", "request": "GET /english/frntpage.htm HTTP/1.1", "status": 200, "size": 12800} +{"@timestamp": 893966423, "clientip":"32.0.0.0", "request": "GET /english/nav_inet.html HTTP/1.1", "status": 200, "size": 2672} +{"@timestamp": 893966423, "clientip":"15.0.0.0", "request": "GET /english/playing/images/backg.gif HTTP/1.0", "status": 200, "size": 1462} +{"@timestamp": 893966423, "clientip":"15.0.0.0", "request": "GET /english/playing/images/play_header.gif HTTP/1.0", "status": 200, "size": 2262} +{"@timestamp": 893966423, "clientip":"20.0.0.0", "request": "GET /images/paris_off.gif HTTP/1.0", "status": 200, "size": 658} +{"@timestamp": 893966423, "clientip":"8.0.0.0", "request": "GET /images/teams_hm_bracket.gif HTTP/1.0", "status": 200, "size": 655} +{"@timestamp": 893966424, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_c.gif HTTP/1.0", "status": 200, "size": 670} +{"@timestamp": 893966424, "clientip":"29.0.0.0", "request": "GET /images/32t49811.jpg HTTP/1.0", "status": 200, "size": 4753} +{"@timestamp": 893966424, "clientip":"37.0.0.0", "request": "GET /images/cal-lens.gif HTTP/1.0", "status": 200, "size": 284} +{"@timestamp": 893966424, "clientip":"40.0.0.0", "request": "GET /english/history/past_cups/images/posters/italy34.gif HTTP/1.1", "status": 200, "size": 4474} +{"@timestamp": 893966424, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_bu_quest2.gif HTTP/1.0", "status": 200, "size": 1201} +{"@timestamp": 893966424, "clientip":"38.0.0.0", "request": "GET /images/s102424.gif HTTP/1.0", "status": 200, "size": 164} +{"@timestamp": 893966424, "clientip":"20.0.0.0", "request": "GET /images/11102.gif HTTP/1.0", "status": 200, "size": 417} +{"@timestamp": 893966424, "clientip":"15.0.0.0", "request": "GET /english/playing/images/france98b.GIF HTTP/1.0", "status": 200, "size": 597} +{"@timestamp": 893966424, "clientip":"39.0.0.0", "request": "GET /english/history/images/france98b.GIF HTTP/1.1", "status": 200, "size": 915} +{"@timestamp": 893966424, "clientip":"39.0.0.0", "request": "GET /english/history/images/football.GIF HTTP/1.1", "status": 200, "size": 1170} +{"@timestamp": 893966424, "clientip":"46.0.0.0", "request": "GET /english/venues/venues/saint-denis.html HTTP/1.0", "status": 200, "size": 22760} +{"@timestamp": 893966425, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_bu_abroad2_on.gif HTTP/1.0", "status": 200, "size": 2020} +{"@timestamp": 893966425, "clientip":"47.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966425, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_abroad_header.gif HTTP/1.0", "status": 200, "size": 3606} +{"@timestamp": 893966425, "clientip":"15.0.0.0", "request": "GET /english/playing/images/mascots.GIF HTTP/1.0", "status": 200, "size": 1275} +{"@timestamp": 893966425, "clientip":"44.0.0.0", "request": "GET /images/10512.jpg HTTP/1.1", "status": 200, "size": 17227} +{"@timestamp": 893966425, "clientip":"44.0.0.0", "request": "GET /images/32t49809.jpg HTTP/1.1", "status": 200, "size": 5623} +{"@timestamp": 893966425, "clientip":"32.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.1", "status": 200, "size": 8389} +{"@timestamp": 893966425, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_bu_infrance2.gif HTTP/1.0", "status": 200, "size": 1134} +{"@timestamp": 893966425, "clientip":"43.0.0.0", "request": "GET /images/32p49815.jpg HTTP/1.0", "status": 200, "size": 11735} +{"@timestamp": 893966425, "clientip":"43.0.0.0", "request": "GET /english/images/team_bu_detail_off.gif HTTP/1.0", "status": 200, "size": 918} +{"@timestamp": 893966425, "clientip":"44.0.0.0", "request": "GET /images/esp.gif HTTP/1.1", "status": 200, "size": 1892} +{"@timestamp": 893966425, "clientip":"38.0.0.0", "request": "GET /english/images/team_group_header_a.gif HTTP/1.0", "status": 200, "size": 639} +{"@timestamp": 893966425, "clientip":"48.0.0.0", "request": "GET /english/help/site.html HTTP/1.0", "status": 200, "size": 7697} +{"@timestamp": 893966425, "clientip":"15.0.0.0", "request": "GET /english/playing/images/downloads.GIF HTTP/1.0", "status": 200, "size": 1294} +{"@timestamp": 893966426, "clientip":"37.0.0.0", "request": "GET /french/images/comp_bu_stage1n.gif HTTP/1.0", "status": 200, "size": 1547} +{"@timestamp": 893966426, "clientip":"25.0.0.0", "request": "GET /french/index.html HTTP/1.1", "status": 200, "size": 954} +{"@timestamp": 893966426, "clientip":"21.0.0.0", "request": "GET /english/images/news_bu_kits_off.gif HTTP/1.1", "status": 200, "size": 1027} +{"@timestamp": 893966426, "clientip":"15.0.0.0", "request": "GET /english/playing/images/links_on.GIF HTTP/1.0", "status": 200, "size": 1393} +{"@timestamp": 893966426, "clientip":"15.0.0.0", "request": "GET /english/playing/images/trivia.GIF HTTP/1.0", "status": 200, "size": 999} +{"@timestamp": 893966426, "clientip":"0.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 200, "size": 892} +{"@timestamp": 893966426, "clientip":"32.0.0.0", "request": "GET /english/nav_top_inet.html HTTP/1.1", "status": 200, "size": 374} +{"@timestamp": 893966427, "clientip":"46.0.0.0", "request": "GET /english/venues/cities/images/denis/venue_denn_bg.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966427, "clientip":"8.0.0.0", "request": "GET /english/images/team_hm_header_shad.gif HTTP/1.0", "status": 200, "size": 1379} +{"@timestamp": 893966427, "clientip":"46.0.0.0", "request": "GET /images/11295.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966427, "clientip":"46.0.0.0", "request": "GET /images/11294.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966427, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_b.gif HTTP/1.0", "status": 200, "size": 665} +{"@timestamp": 893966427, "clientip":"38.0.0.0", "request": "GET /images/s102330.gif HTTP/1.0", "status": 200, "size": 259} +{"@timestamp": 893966427, "clientip":"15.0.0.0", "request": "GET /english/playing/images/banner2.gif HTTP/1.0", "status": 200, "size": 15328} +{"@timestamp": 893966427, "clientip":"49.0.0.0", "request": "GET /english/news/2704nevi.htm HTTP/1.0", "status": 200, "size": 3132} +{"@timestamp": 893966427, "clientip":"38.0.0.0", "request": "GET /english/images/team_group_header_b.gif HTTP/1.0", "status": 200, "size": 665} +{"@timestamp": 893966427, "clientip":"15.0.0.0", "request": "GET /english/playing/images/fifa_logo_sm.gif HTTP/1.0", "status": 200, "size": 2900} +{"@timestamp": 893966427, "clientip":"39.0.0.0", "request": "GET /english/history/images/infrance.GIF HTTP/1.1", "status": 200, "size": 990} +{"@timestamp": 893966427, "clientip":"47.135.0.0", "request": "GET /french/news/3004bres.htm HTTP/1.0", "status": 200, "size": 5933} +{"@timestamp": 893966428, "clientip":"50.0.0.0", "request": "GET /english/playing/download/images/big.bird.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966428, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_e.gif HTTP/1.0", "status": 200, "size": 643} +{"@timestamp": 893966429, "clientip":"41.0.0.0", "request": "GET /english/images/team_bu_detail_on.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966429, "clientip":"30.0.0.0", "request": "GET /english/playing/mascot/images/misc.gif HTTP/1.0", "status": 200, "size": 2997} +{"@timestamp": 893966429, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_g.gif HTTP/1.0", "status": 200, "size": 641} +{"@timestamp": 893966429, "clientip":"38.0.0.0", "request": "GET /images/s102382.gif HTTP/1.0", "status": 200, "size": 102} +{"@timestamp": 893966429, "clientip":"12.0.0.0", "request": "GET /english/tickets/10484.htm HTTP/1.0", "status": 200, "size": 16722} +{"@timestamp": 893966429, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_h.gif HTTP/1.0", "status": 200, "size": 628} +{"@timestamp": 893966429, "clientip":"12.0.0.0", "request": "GET /english/tickets/images/ticket_bu_abroad2.gif HTTP/1.0", "status": 200, "size": 1486} +{"@timestamp": 893966430, "clientip":"12.0.0.0", "request": "GET /images/11103.gif HTTP/1.0", "status": 200, "size": 513} +{"@timestamp": 893966430, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_d.gif HTTP/1.0", "status": 200, "size": 646} +{"@timestamp": 893966430, "clientip":"5.0.0.0", "request": "GET /french/images/archives.gif HTTP/1.1", "status": 200, "size": 569} +{"@timestamp": 893966430, "clientip":"39.0.0.0", "request": "GET /english/history/images/reading.GIF HTTP/1.1", "status": 200, "size": 1171} +{"@timestamp": 893966430, "clientip":"32.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.1", "status": 200, "size": 6507} +{"@timestamp": 893966430, "clientip":"8.0.0.0", "request": "GET /english/images/team_group_header_f.gif HTTP/1.0", "status": 200, "size": 631} +{"@timestamp": 893966430, "clientip":"26.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966430, "clientip":"47.135.0.0", "request": "GET /images/bord_g.gif HTTP/1.0", "status": 200, "size": 231} +{"@timestamp": 893966430, "clientip":"47.135.0.0", "request": "GET /images/bord_d.gif HTTP/1.0", "status": 200, "size": 231} +{"@timestamp": 893966431, "clientip":"12.0.0.0", "request": "GET /images/10982.gif HTTP/1.0", "status": 200, "size": 183} +{"@timestamp": 893966431, "clientip":"38.0.0.0", "request": "GET /images/s102373.gif HTTP/1.0", "status": 200, "size": 142} +{"@timestamp": 893966431, "clientip":"38.0.0.0", "request": "GET /images/s102328.gif HTTP/1.0", "status": 200, "size": 236} +{"@timestamp": 893966431, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_header.gif HTTP/1.1", "status": 200, "size": 1989} +{"@timestamp": 893966431, "clientip":"51.0.0.0", "request": "GET /english/teams/teambio160.htm HTTP/1.1", "status": 200, "size": 11097} +{"@timestamp": 893966431, "clientip":"38.0.0.0", "request": "GET /english/images/team_group_header_f.gif HTTP/1.0", "status": 200, "size": 631} +{"@timestamp": 893966431, "clientip":"52.0.0.0", "request": "GET /english/member/body.html HTTP/1.1", "status": 200, "size": 5097} +{"@timestamp": 893966432, "clientip":"10.0.0.0", "request": "GET /english/competition/stage2.htm HTTP/1.0", "status": 200, "size": 16606} +{"@timestamp": 893966432, "clientip":"0.0.0.0", "request": "GET /english/nav_top_inet.html HTTP/1.0", "status": 200, "size": 374} +{"@timestamp": 893966432, "clientip":"53.0.0.0", "request": "GET /english/competition/headtohead75.htm HTTP/1.1", "status": 200, "size": 5152} +{"@timestamp": 893966432, "clientip":"25.0.0.0", "request": "GET /french/nav_top_inet.html HTTP/1.1", "status": 200, "size": 374} +{"@timestamp": 893966432, "clientip":"0.0.0.0", "request": "GET /english/nav_inet.html HTTP/1.0", "status": 200, "size": 2672} +{"@timestamp": 893966432, "clientip":"0.0.0.0", "request": "GET /english/splash_inet.html HTTP/1.0", "status": 200, "size": 3730} +{"@timestamp": 893966432, "clientip":"39.0.0.0", "request": "GET /english/history/images/history_hm_header.gif HTTP/1.1", "status": 200, "size": 688} +{"@timestamp": 893966432, "clientip":"47.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966432, "clientip":"37.0.0.0", "request": "GET /images/cal_nant.gif HTTP/1.0", "status": 200, "size": 359} +{"@timestamp": 893966432, "clientip":"37.0.0.0", "request": "GET /images/cal_stdenis.gif HTTP/1.0", "status": 200, "size": 402} +{"@timestamp": 893966432, "clientip":"41.0.0.0", "request": "GET /images/32t49807.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966433, "clientip":"32.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.1", "status": 200, "size": 853} +{"@timestamp": 893966433, "clientip":"38.0.0.0", "request": "GET /images/s102320.gif HTTP/1.0", "status": 200, "size": 259} +{"@timestamp": 893966433, "clientip":"42.0.0.0", "request": "GET /images/backnews.gif HTTP/1.0", "status": 200, "size": 4573} +{"@timestamp": 893966433, "clientip":"38.0.0.0", "request": "GET /images/s102438.gif HTTP/1.0", "status": 200, "size": 297} +{"@timestamp": 893966433, "clientip":"54.0.0.0", "request": "GET /english/playing/download/images/big.bird.gif HTTP/1.0", "status": 200, "size": 4870} +{"@timestamp": 893966433, "clientip":"0.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.0", "status": 200, "size": 853} +{"@timestamp": 893966433, "clientip":"38.0.0.0", "request": "GET /images/s102329.gif HTTP/1.0", "status": 200, "size": 159} +{"@timestamp": 893966433, "clientip":"10.0.0.0", "request": "GET /images/comp_stage2_brc_bot.gif HTTP/1.0", "status": 200, "size": 160} +{"@timestamp": 893966434, "clientip":"26.0.0.0", "request": "GET /english/images/nav_home_off.gif HTTP/1.0", "status": 200, "size": 828} +{"@timestamp": 893966434, "clientip":"0.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966434, "clientip":"41.0.0.0", "request": "GET /english/images/team_hm_header.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966434, "clientip":"32.0.0.0", "request": "GET /english/images/nav_team_off.gif HTTP/1.1", "status": 200, "size": 776} +{"@timestamp": 893966434, "clientip":"38.0.0.0", "request": "GET /english/images/team_group_header_c.gif HTTP/1.0", "status": 200, "size": 670} +{"@timestamp": 893966434, "clientip":"52.0.0.0", "request": "GET /english/member/images/submit.gif HTTP/1.1", "status": 200, "size": 447} +{"@timestamp": 893966434, "clientip":"12.0.0.0", "request": "GET /images/10981.gif HTTP/1.0", "status": 200, "size": 173} +{"@timestamp": 893966434, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_34_off.gif HTTP/1.1", "status": 200, "size": 397} +{"@timestamp": 893966434, "clientip":"38.0.0.0", "request": "GET /images/s102377.gif HTTP/1.0", "status": 200, "size": 173} +{"@timestamp": 893966434, "clientip":"12.0.0.0", "request": "GET /images/11116.gif HTTP/1.0", "status": 200, "size": 667} +{"@timestamp": 893966435, "clientip":"26.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966435, "clientip":"12.0.0.0", "request": "GET /images/11106.gif HTTP/1.0", "status": 200, "size": 114} +{"@timestamp": 893966435, "clientip":"12.0.0.0", "request": "GET /images/11105.gif HTTP/1.0", "status": 200, "size": 114} +{"@timestamp": 893966435, "clientip":"12.0.0.0", "request": "GET /images/11102.gif HTTP/1.0", "status": 200, "size": 417} +{"@timestamp": 893966435, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_38_off.gif HTTP/1.1", "status": 200, "size": 436} +{"@timestamp": 893966435, "clientip":"38.0.0.0", "request": "GET /english/images/team_group_header_h.gif HTTP/1.0", "status": 200, "size": 628} +{"@timestamp": 893966435, "clientip":"25.0.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.1", "status": 200, "size": 929} +{"@timestamp": 893966435, "clientip":"52.0.0.0", "request": "GET / HTTP/1.1", "status": 200, "size": 8712} +{"@timestamp": 893966435, "clientip":"12.0.0.0", "request": "GET /images/11101.gif HTTP/1.0", "status": 200, "size": 415} +{"@timestamp": 893966435, "clientip":"32.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.1", "status": 200, "size": 870} +{"@timestamp": 893966436, "clientip":"55.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966436, "clientip":"56.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.0", "status": 200, "size": 60349} +{"@timestamp": 893966436, "clientip":"38.0.0.0", "request": "GET /images/s102443.gif HTTP/1.0", "status": 200, "size": 165} +{"@timestamp": 893966436, "clientip":"46.0.0.0", "request": "GET /images/paris.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966436, "clientip":"38.0.0.0", "request": "GET /images/s140875.gif HTTP/1.0", "status": 200, "size": 184} +{"@timestamp": 893966436, "clientip":"37.0.0.0", "request": "GET /images/cal_bord.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966436, "clientip":"37.0.0.0", "request": "GET /images/cal_steti.gif HTTP/1.0", "status": 200, "size": 1125} +{"@timestamp": 893966436, "clientip":"37.0.0.0", "request": "GET /images/cal_mont.gif HTTP/1.0", "status": 200, "size": 316} +{"@timestamp": 893966436, "clientip":"51.0.0.0", "request": "GET /images/32t49813.jpg HTTP/1.1", "status": 200, "size": 4714} +{"@timestamp": 893966436, "clientip":"2.0.0.0", "request": "GET /english/images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966437, "clientip":"23.0.0.0", "request": "GET /english/tickets/body.html HTTP/1.0", "status": 200, "size": 2925} +{"@timestamp": 893966437, "clientip":"52.0.0.0", "request": "GET /english/member/images/cfologo.gif HTTP/1.1", "status": 200, "size": 2820} +{"@timestamp": 893966437, "clientip":"52.0.0.0", "request": "GET /english/member/images/member_header.jpg HTTP/1.1", "status": 200, "size": 10457} +{"@timestamp": 893966437, "clientip":"32.0.0.0", "request": "GET /english/news/3004bres.htm HTTP/1.1", "status": 200, "size": 5765} +{"@timestamp": 893966437, "clientip":"38.0.0.0", "request": "GET /images/s102386.gif HTTP/1.0", "status": 200, "size": 279} +{"@timestamp": 893966437, "clientip":"41.0.0.0", "request": "GET /images/news_hm_arw.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966437, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_54_off.gif HTTP/1.1", "status": 200, "size": 403} +{"@timestamp": 893966437, "clientip":"38.0.0.0", "request": "GET /images/s102326.gif HTTP/1.0", "status": 200, "size": 248} +{"@timestamp": 893966437, "clientip":"23.0.0.0", "request": "GET /english/tickets/images/ticket_hm_bg.jpg HTTP/1.0", "status": 200, "size": 20929} +{"@timestamp": 893966437, "clientip":"23.0.0.0", "request": "GET /english/tickets/images/ticket_hm_header.gif HTTP/1.0", "status": 200, "size": 1226} +{"@timestamp": 893966437, "clientip":"23.0.0.0", "request": "GET /english/tickets/images/ticket_hm_nav.gif HTTP/1.0", "status": 200, "size": 11253} +{"@timestamp": 893966438, "clientip":"33.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.1", "status": 200, "size": 327} +{"@timestamp": 893966438, "clientip":"38.0.0.0", "request": "GET /english/competition/maincomp.htm HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966438, "clientip":"46.0.0.0", "request": "GET /images/11291.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966438, "clientip":"47.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966438, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_58_off.gif HTTP/1.1", "status": 200, "size": 397} +{"@timestamp": 893966439, "clientip":"25.0.0.0", "request": "GET /french/images/nav_news_off.gif HTTP/1.1", "status": 200, "size": 855} +{"@timestamp": 893966439, "clientip":"57.0.0.0", "request": "GET /images/hm_f98_top.gif HTTP/1.0", "status": 200, "size": 915} +{"@timestamp": 893966439, "clientip":"47.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966439, "clientip":"58.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893966439, "clientip":"57.0.0.0", "request": "GET /images/ligne1_case5.gif HTTP/1.0", "status": 200, "size": 1018} +{"@timestamp": 893966439, "clientip":"57.0.0.0", "request": "GET /english/images/top_stories.gif HTTP/1.0", "status": 200, "size": 1210} +{"@timestamp": 893966440, "clientip":"47.0.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966440, "clientip":"47.0.0.0", "request": "GET /images/hm_linkf.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966440, "clientip":"38.0.0.0", "request": "GET /english/images/team_hm_header_shad.gif HTTP/1.0", "status": 200, "size": 1379} +{"@timestamp": 893966440, "clientip":"38.0.0.0", "request": "GET /english/images/comp_hm_nav.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966441, "clientip":"25.0.0.0", "request": "GET /french/images/nav_venue_off.gif HTTP/1.1", "status": 200, "size": 945} +{"@timestamp": 893966441, "clientip":"47.0.0.0", "request": "GET /images/info.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966441, "clientip":"59.0.0.0", "request": "GET /english/news/2704leag.htm HTTP/1.0", "status": 200, "size": 4801} +{"@timestamp": 893966441, "clientip":"60.0.0.0", "request": "GET /images/10531.jpg HTTP/1.0", "status": 200, "size": 16554} +{"@timestamp": 893966441, "clientip":"47.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966441, "clientip":"47.0.0.0", "request": "GET /images/dot.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966441, "clientip":"47.0.0.0", "request": "GET /english/images/nav_home_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966441, "clientip":"47.0.0.0", "request": "GET /images/hm_day_e.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966441, "clientip":"47.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966442, "clientip":"61.0.0.0", "request": "GET /english/venues/cities/montpellier.html HTTP/1.0", "status": 200, "size": 9686} +{"@timestamp": 893966442, "clientip":"62.0.0.0", "request": "GET /english/playing/links.html HTTP/1.0", "status": 200, "size": 18694} +{"@timestamp": 893966442, "clientip":"23.0.0.0", "request": "GET /images/arw_lk.gif HTTP/1.0", "status": 200, "size": 669} +{"@timestamp": 893966442, "clientip":"63.0.0.0", "request": "GET /images/home_bg_stars.gif HTTP/1.0", "status": 200, "size": 2557} +{"@timestamp": 893966442, "clientip":"63.0.0.0", "request": "GET /images/home_fr_phrase.gif HTTP/1.0", "status": 200, "size": 2843} +{"@timestamp": 893966442, "clientip":"63.0.0.0", "request": "GET /images/home_logo.gif HTTP/1.0", "status": 200, "size": 3401} +{"@timestamp": 893966442, "clientip":"47.0.0.0", "request": "GET /english/images/nav_tickets_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966442, "clientip":"47.0.0.0", "request": "GET /english/images/nav_team_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966442, "clientip":"46.0.0.0", "request": "GET /images/sdffr.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966442, "clientip":"47.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966442, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_62_off.gif HTTP/1.1", "status": 200, "size": 437} +{"@timestamp": 893966442, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_66_off.gif HTTP/1.1", "status": 200, "size": 435} +{"@timestamp": 893966443, "clientip":"64.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.0", "status": 200, "size": 327} +{"@timestamp": 893966443, "clientip":"38.0.0.0", "request": "GET /english/images/comp_hm_nav.gif HTTP/1.0", "status": 206, "size": 10902} +{"@timestamp": 893966443, "clientip":"25.0.0.0", "request": "GET /french/images/nav_tickets_off.gif HTTP/1.1", "status": 200, "size": 965} +{"@timestamp": 893966443, "clientip":"57.0.0.0", "request": "GET /images/case5.gif HTTP/1.0", "status": 200, "size": 1362} +{"@timestamp": 893966443, "clientip":"65.0.0.0", "request": "GET /english/index.html HTTP/1.0", "status": 200, "size": 892} +{"@timestamp": 893966443, "clientip":"30.0.0.0", "request": "GET /english/playing/mascot/images/button.03.gif HTTP/1.0", "status": 200, "size": 893} +{"@timestamp": 893966443, "clientip":"57.0.0.0", "request": "GET /images/bord_stories.gif HTTP/1.0", "status": 200, "size": 520} +{"@timestamp": 893966443, "clientip":"57.0.0.0", "request": "GET /images/bord_stories01.gif HTTP/1.0", "status": 200, "size": 333} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /english/nav_inet.html HTTP/1.0", "status": 200, "size": 2672} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /english/splash_inet.html HTTP/1.0", "status": 200, "size": 3730} +{"@timestamp": 893966444, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_70_off.gif HTTP/1.1", "status": 200, "size": 409} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /english/images/nav_team_off.gif HTTP/1.0", "status": 200, "size": 776} +{"@timestamp": 893966444, "clientip":"6.0.0.0", "request": "GET /french/venues/body.html HTTP/1.0", "status": 200, "size": 2042} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.0", "status": 200, "size": 853} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893966444, "clientip":"64.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.0", "status": 200, "size": 2491} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.0", "status": 200, "size": 33665} +{"@timestamp": 893966444, "clientip":"65.0.0.0", "request": "GET /english/images/nav_comp_off.gif HTTP/1.0", "status": 200, "size": 994} +{"@timestamp": 893966444, "clientip":"66.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966444, "clientip":"62.0.0.0", "request": "GET /english/playing/images/backg.gif HTTP/1.0", "status": 200, "size": 1462} +{"@timestamp": 893966444, "clientip":"67.0.0.0", "request": "GET /english/individuals/player389.htm HTTP/1.0", "status": 200, "size": 7001} +{"@timestamp": 893966444, "clientip":"47.0.0.0", "request": "GET /english/teams/teamgroup.htm HTTP/1.0", "status": 200, "size": 11971} +{"@timestamp": 893966445, "clientip":"7.0.0.0", "request": "GET /english/playing/images/play_header.gif HTTP/1.0", "status": 200, "size": 2262} +{"@timestamp": 893966445, "clientip":"30.0.0.0", "request": "GET /english/playing/mascot/images/footix.test.gif HTTP/1.0", "status": 200, "size": 20385} +{"@timestamp": 893966445, "clientip":"6.0.0.0", "request": "GET /french/venues/images/Venue_map_mid_off.gif HTTP/1.0", "status": 200, "size": 9911} +{"@timestamp": 893966445, "clientip":"25.0.0.0", "request": "GET /french/images/nav_field_off.gif HTTP/1.1", "status": 200, "size": 982} +{"@timestamp": 893966445, "clientip":"64.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.0", "status": 200, "size": 60349} +{"@timestamp": 893966445, "clientip":"6.0.0.0", "request": "GET /french/venues/images/Venue_map_top_off.gif HTTP/1.0", "status": 200, "size": 8369} +{"@timestamp": 893966445, "clientip":"6.0.0.0", "request": "GET /french/venues/images/venue_hm_nav.gif HTTP/1.0", "status": 200, "size": 7644} +{"@timestamp": 893966445, "clientip":"68.0.0.0", "request": "GET /english/venues/cities/images/montpellier/venue_mont_header.jpg HTTP/1.0", "status": 200, "size": 11562} +{"@timestamp": 893966445, "clientip":"69.0.0.0", "request": "GET /english/history/history_of/images/cup/cup.gif HTTP/1.0", "status": 200, "size": 5469} +{"@timestamp": 893966445, "clientip":"34.0.0.0", "request": "GET /english/history/body.html HTTP/1.0", "status": 200, "size": 2909} +{"@timestamp": 893966445, "clientip":"70.0.0.0", "request": "GET /english/venues/cities/images/marseille/mars_c.gif HTTP/1.0", "status": 200, "size": 369} +{"@timestamp": 893966446, "clientip":"64.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.0", "status": 200, "size": 2140} +{"@timestamp": 893966446, "clientip":"66.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966446, "clientip":"6.0.0.0", "request": "GET /french/venues/images/Venue_map_bot_off.gif HTTP/1.0", "status": 200, "size": 7397} +{"@timestamp": 893966446, "clientip":"71.0.0.0", "request": "GET /english/teams/teambio148.htm HTTP/1.0", "status": 200, "size": 10857} +{"@timestamp": 893966446, "clientip":"72.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966446, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_78_off.gif HTTP/1.1", "status": 200, "size": 427} +{"@timestamp": 893966446, "clientip":"67.0.0.0", "request": "GET /images/102373.gif HTTP/1.0", "status": 200, "size": 1703} +{"@timestamp": 893966446, "clientip":"47.0.0.0", "request": "GET /english/images/teams_bu_group_on.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966446, "clientip":"67.0.0.0", "request": "GET /images/32p49804.jpg HTTP/1.0", "status": 200, "size": 11023} +{"@timestamp": 893966446, "clientip":"47.0.0.0", "request": "GET /english/images/teams_bu_confed_off.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966447, "clientip":"71.0.0.0", "request": "GET /images/32t49812.jpg HTTP/1.0", "status": 200, "size": 4132} +{"@timestamp": 893966447, "clientip":"71.0.0.0", "request": "GET /images/ger.gif HTTP/1.0", "status": 200, "size": 1603} +{"@timestamp": 893966447, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_82_off.gif HTTP/1.1", "status": 200, "size": 433} +{"@timestamp": 893966447, "clientip":"25.0.0.0", "request": "GET /french/images/nav_store_off.gif HTTP/1.1", "status": 200, "size": 976} +{"@timestamp": 893966447, "clientip":"65.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966447, "clientip":"65.0.0.0", "request": "GET /english/images/hm_official.gif HTTP/1.0", "status": 200, "size": 1807} +{"@timestamp": 893966447, "clientip":"47.0.0.0", "request": "GET /images/s102325.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966447, "clientip":"65.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966447, "clientip":"47.0.0.0", "request": "GET /english/images/team_group_header_a.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966447, "clientip":"65.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 1005} +{"@timestamp": 893966447, "clientip":"65.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966448, "clientip":"65.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 200, "size": 6507} +{"@timestamp": 893966448, "clientip":"47.0.0.0", "request": "GET /images/s102424.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966448, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_86_off.gif HTTP/1.1", "status": 200, "size": 409} +{"@timestamp": 893966448, "clientip":"47.0.0.0", "request": "GET /images/s102487.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966448, "clientip":"65.0.0.0", "request": "GET /english/images/nav_hosts_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966448, "clientip":"65.0.0.0", "request": "GET /images/hm_brdl.gif HTTP/1.0", "status": 200, "size": 208} +{"@timestamp": 893966448, "clientip":"73.0.0.0", "request": "GET / HTTP/1.1", "status": 304, "size": 0} +{"@timestamp": 893966448, "clientip":"62.0.0.0", "request": "GET /english/playing/images/links_on.GIF HTTP/1.0", "status": 200, "size": 1393} +{"@timestamp": 893966448, "clientip":"62.0.0.0", "request": "GET /english/playing/images/banner2.gif HTTP/1.0", "status": 200, "size": 15328} +{"@timestamp": 893966448, "clientip":"66.0.0.0", "request": "GET /images/home_bg_stars.gif HTTP/1.0", "status": 200, "size": 2557} +{"@timestamp": 893966448, "clientip":"57.0.0.0", "request": "GET /english/images/lateb_new.gif HTTP/1.0", "status": 200, "size": 1431} +{"@timestamp": 893966448, "clientip":"47.0.0.0", "request": "GET /images/s102477.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966448, "clientip":"65.0.0.0", "request": "GET /english/images/nav_sitemap_off.gif HTTP/1.0", "status": 200, "size": 416} +{"@timestamp": 893966448, "clientip":"65.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966448, "clientip":"47.0.0.0", "request": "GET /english/images/team_group_header_e.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966448, "clientip":"65.0.0.0", "request": "GET /images/hm_linkf.gif HTTP/1.0", "status": 200, "size": 123} +{"@timestamp": 893966449, "clientip":"65.0.0.0", "request": "GET /images/info.gif HTTP/1.0", "status": 200, "size": 1251} +{"@timestamp": 893966449, "clientip":"74.0.0.0", "request": "GET / HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966449, "clientip":"75.0.0.0", "request": "GET /english/competition/stage2.htm HTTP/1.0", "status": 200, "size": 16606} +{"@timestamp": 893966449, "clientip":"47.0.0.0", "request": "GET /images/s102324.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966449, "clientip":"25.0.0.0", "request": "GET /french/images/nav_sitemap_off.gif HTTP/1.1", "status": 200, "size": 413} +{"@timestamp": 893966449, "clientip":"65.0.0.0", "request": "GET /images/hm_day_e.gif HTTP/1.0", "status": 200, "size": 499} +{"@timestamp": 893966449, "clientip":"14.0.0.0", "request": "GET /images/home_bg_stars.gif HTTP/1.1", "status": 200, "size": 2557} +{"@timestamp": 893966449, "clientip":"47.0.0.0", "request": "GET /images/s102376.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966449, "clientip":"40.0.0.0", "request": "GET /english/history/past_cups/france38.html HTTP/1.1", "status": 200, "size": 12669} +{"@timestamp": 893966449, "clientip":"73.0.0.0", "request": "GET /images/home_fr_phrase.gif HTTP/1.1", "status": 200, "size": 2843} +{"@timestamp": 893966449, "clientip":"73.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.1", "status": 200, "size": 60349} +{"@timestamp": 893966449, "clientip":"65.0.0.0", "request": "GET /images/hm_brdr.gif HTTP/1.0", "status": 200, "size": 235} +{"@timestamp": 893966449, "clientip":"65.0.0.0", "request": "GET /english/images/nav_store_off.gif HTTP/1.0", "status": 200, "size": 934} +{"@timestamp": 893966449, "clientip":"76.0.0.0", "request": "GET /english/competition/stage1.htm HTTP/1.1", "status": 200, "size": 36783} +{"@timestamp": 893966449, "clientip":"65.0.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966449, "clientip":"53.0.0.0", "request": "GET /english/competition/headtohead76.htm HTTP/1.1", "status": 200, "size": 16909} +{"@timestamp": 893966449, "clientip":"65.0.0.0", "request": "GET /images/dot.gif HTTP/1.0", "status": 200, "size": 43} +{"@timestamp": 893966449, "clientip":"2.0.0.0", "request": "GET /english/images/nav_history_off.gif HTTP/1.0", "status": 200, "size": 914} +{"@timestamp": 893966449, "clientip":"73.0.0.0", "request": "GET /images/home_eng_phrase.gif HTTP/1.1", "status": 200, "size": 2861} +{"@timestamp": 893966450, "clientip":"2.0.0.0", "request": "GET /english/images/nav_field_off.gif HTTP/1.0", "status": 200, "size": 1005} +{"@timestamp": 893966450, "clientip":"66.0.0.0", "request": "GET /images/home_fr_phrase.gif HTTP/1.0", "status": 200, "size": 2843} +{"@timestamp": 893966450, "clientip":"1.0.0.0", "request": "GET /english/images/hm_official.gif HTTP/1.0", "status": 200, "size": 1807} +{"@timestamp": 893966450, "clientip":"1.0.0.0", "request": "GET /images/nav_bg_top.gif HTTP/1.0", "status": 200, "size": 929} +{"@timestamp": 893966450, "clientip":"66.0.0.0", "request": "GET /images/home_intro.anim.gif HTTP/1.0", "status": 200, "size": 60349} +{"@timestamp": 893966450, "clientip":"1.0.0.0", "request": "GET /images/logo_cfo.gif HTTP/1.0", "status": 200, "size": 1504} +{"@timestamp": 893966450, "clientip":"1.0.0.0", "request": "GET /images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966450, "clientip":"77.0.0.0", "request": "GET /english/images/france98b.gif HTTP/1.0", "status": 200, "size": 2122} +{"@timestamp": 893966450, "clientip":"1.0.0.0", "request": "GET /english/ProScroll.class HTTP/1.0", "status": 200, "size": 6507} +{"@timestamp": 893966450, "clientip":"77.0.0.0", "request": "GET /english/images/space.gif HTTP/1.0", "status": 200, "size": 42} +{"@timestamp": 893966450, "clientip":"57.0.0.0", "request": "GET /images/ligne4_latebreak.gif HTTP/1.0", "status": 200, "size": 1056} +{"@timestamp": 893966450, "clientip":"47.0.0.0", "request": "GET /images/s102373.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966450, "clientip":"47.0.0.0", "request": "GET /images/s102443.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966450, "clientip":"232.0.0.0", "request": "GET /images/hm_bg.jpg HTTP/1.0", "status": 200, "size": 24736} +{"@timestamp": 893966451, "clientip":"66.0.0.0", "request": "GET /images/home_eng_phrase.gif HTTP/1.0", "status": 200, "size": 2861} +{"@timestamp": 893966451, "clientip":"75.0.0.0", "request": "GET /english/images/comp_bu_stage2n_on.gif HTTP/1.0", "status": 200, "size": 996} +{"@timestamp": 893966451, "clientip":"57.0.0.0", "request": "GET /images/ligneb.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966451, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bu_94_off.gif HTTP/1.1", "status": 200, "size": 432} +{"@timestamp": 893966451, "clientip":"77.0.0.0", "request": "GET /english/images/comp_bu_stage1n.gif HTTP/1.0", "status": 200, "size": 1548} +{"@timestamp": 893966451, "clientip":"47.0.0.0", "request": "GET /images/s140875.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966451, "clientip":"47.0.0.0", "request": "GET /images/s102321.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966451, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/venue_denn_bg.jpg HTTP/1.0", "status": 200, "size": 21003} +{"@timestamp": 893966451, "clientip":"6.0.0.0", "request": "GET /french/venues/images/venue_header.gif HTTP/1.0", "status": 200, "size": 740} +{"@timestamp": 893966451, "clientip":"75.0.0.0", "request": "GET /images/comp_stage2_brc_top.gif HTTP/1.0", "status": 200, "size": 163} +{"@timestamp": 893966451, "clientip":"75.0.0.0", "request": "GET /images/comp_stage2_brc_topr.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966451, "clientip":"25.0.0.0", "request": "GET /images/hm_nbg.jpg HTTP/1.1", "status": 200, "size": 33665} +{"@timestamp": 893966451, "clientip":"6.0.0.0", "request": "GET /french/venues/images/venue_bu_city_on.gif HTTP/1.0", "status": 200, "size": 1061} +{"@timestamp": 893966451, "clientip":"66.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.0", "status": 200, "size": 327} +{"@timestamp": 893966451, "clientip":"47.0.0.0", "request": "GET /english/images/team_group_header_g.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966451, "clientip":"47.0.0.0", "request": "GET /images/s102357.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966451, "clientip":"75.0.0.0", "request": "GET /images/comp_stage2_brc_bot.gif HTTP/1.0", "status": 200, "size": 160} +{"@timestamp": 893966451, "clientip":"69.0.0.0", "request": "GET /english/history/history_of/images/cup/coup_du_mondtxt.gif HTTP/1.0", "status": 200, "size": 5053} +{"@timestamp": 893966451, "clientip":"73.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.1", "status": 200, "size": 2140} +{"@timestamp": 893966451, "clientip":"1.0.0.0", "request": "GET /images/nav_bg_bottom.jpg HTTP/1.0", "status": 200, "size": 8389} +{"@timestamp": 893966451, "clientip":"1.0.0.0", "request": "GET /english/images/nav_news_off.gif HTTP/1.0", "status": 200, "size": 853} +{"@timestamp": 893966451, "clientip":"1.0.0.0", "request": "GET /english/images/nav_comp_off.gif HTTP/1.0", "status": 200, "size": 994} +{"@timestamp": 893966451, "clientip":"47.0.0.0", "request": "GET /english/images/team_hm_header_shad.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966452, "clientip":"47.0.0.0", "request": "GET /english/images/team_group_header_c.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966452, "clientip":"78.0.0.0", "request": "GET /english/venues/cities/images/montpellier/12eglise.jpg HTTP/1.0", "status": 200, "size": 3035} +{"@timestamp": 893966452, "clientip":"47.0.0.0", "request": "GET /images/s102383.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966452, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/12maisons.jpg HTTP/1.0", "status": 200, "size": 2559} +{"@timestamp": 893966452, "clientip":"75.0.0.0", "request": "GET /images/comp_stage2_brc_botr.gif HTTP/1.0", "status": 200, "size": 158} +{"@timestamp": 893966452, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/denis_a.gif HTTP/1.0", "status": 200, "size": 520} +{"@timestamp": 893966452, "clientip":"70.0.0.0", "request": "GET /english/venues/cities/images/montpellier/17mystery.gif HTTP/1.0", "status": 200, "size": 542} +{"@timestamp": 893966452, "clientip":"79.0.0.0", "request": "GET /english/history/images/history_hm_bg2.jpg HTTP/1.0", "status": 200, "size": 22515} +{"@timestamp": 893966452, "clientip":"79.0.0.0", "request": "GET /english/history/images/history_hm_header.gif HTTP/1.0", "status": 200, "size": 688} +{"@timestamp": 893966452, "clientip":"2.0.0.0", "request": "GET /english/images/nav_home_off.gif HTTP/1.0", "status": 200, "size": 828} +{"@timestamp": 893966452, "clientip":"1.0.0.0", "request": "GET /images/hm_day_e.gif HTTP/1.0", "status": 200, "size": 499} +{"@timestamp": 893966452, "clientip":"70.0.0.0", "request": "GET /english/venues/cities/images/montpellier/18corum.gif HTTP/1.0", "status": 200, "size": 650} +{"@timestamp": 893966452, "clientip":"66.0.0.0", "request": "GET /images/home_eng_button.gif HTTP/1.0", "status": 200, "size": 1927} +{"@timestamp": 893966452, "clientip":"80.0.0.0", "request": "GET /english/news/player01.htm HTTP/1.0", "status": 200, "size": 42591} +{"@timestamp": 893966452, "clientip":"79.0.0.0", "request": "GET /english/history/images/history_hm_3094.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966452, "clientip":"79.0.0.0", "request": "GET /english/history/images/history_hm_posters.jpg HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966452, "clientip":"39.0.0.0", "request": "GET /english/history/past_cups/images/past_bracket_top.gif HTTP/1.1", "status": 200, "size": 289} +{"@timestamp": 893966452, "clientip":"2.0.0.0", "request": "GET /english/images/nav_tickets_off.gif HTTP/1.0", "status": 200, "size": 937} +{"@timestamp": 893966452, "clientip":"47.0.0.0", "request": "GET /images/s102380.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966452, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/16tradition.gif HTTP/1.0", "status": 200, "size": 387} +{"@timestamp": 893966452, "clientip":"66.0.0.0", "request": "GET /images/home_sponsor.gif HTTP/1.0", "status": 200, "size": 2491} +{"@timestamp": 893966453, "clientip":"70.0.0.0", "request": "GET /english/venues/cities/images/marseille/mars_t.gif HTTP/1.0", "status": 200, "size": 353} +{"@timestamp": 893966453, "clientip":"73.0.0.0", "request": "GET /images/home_tool.gif HTTP/1.1", "status": 200, "size": 327} +{"@timestamp": 893966453, "clientip":"81.0.0.0", "request": "GET / HTTP/1.0", "status": 200, "size": 8712} +{"@timestamp": 893966453, "clientip":"5.0.0.0", "request": "GET /french/news/3004arge.htm HTTP/1.1", "status": 200, "size": 4869} +{"@timestamp": 893966453, "clientip":"78.0.0.0", "request": "GET /english/venues/cities/images/marseille/mars_i.gif HTTP/1.0", "status": 200, "size": 314} +{"@timestamp": 893966453, "clientip":"1.0.0.0", "request": "GET /images/hm_arw.gif HTTP/1.0", "status": 200, "size": 1050} +{"@timestamp": 893966453, "clientip":"77.0.0.0", "request": "GET /english/images/comp_bu_stage2n.gif HTTP/1.0", "status": 200, "size": 984} +{"@timestamp": 893966453, "clientip":"2.0.0.0", "request": "GET /english/images/nav_logo_sponsors.gif HTTP/1.0", "status": 200, "size": 1991} +{"@timestamp": 893966453, "clientip":"62.0.0.0", "request": "GET /english/playing/images/fifa_logo_sm.gif HTTP/1.0", "status": 200, "size": 2900} +{"@timestamp": 893966453, "clientip":"68.0.0.0", "request": "GET /english/venues/cities/images/marseille/mars_a.gif HTTP/1.0", "status": 200, "size": 434} +{"@timestamp": 893966453, "clientip":"47.0.0.0", "request": "GET /images/s102320.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966453, "clientip":"3.0.0.0", "request": "GET /english/images/fpnewstop.gif HTTP/1.0", "status": 200, "size": 568} +{"@timestamp": 893966453, "clientip":"1.0.0.0", "request": "GET /english/images/nav_venue_off.gif HTTP/1.0", "status": 200, "size": 870} +{"@timestamp": 893966453, "clientip":"3.0.0.0", "request": "GET /english/images/news_btn_letter_off.gif HTTP/1.0", "status": 200, "size": 852} +{"@timestamp": 893966453, "clientip":"47.0.0.0", "request": "GET /images/s102353.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966453, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/17commercial.gif HTTP/1.0", "status": 200, "size": 614} +{"@timestamp": 893966453, "clientip":"57.0.0.0", "request": "GET /images/ligne.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966453, "clientip":"47.0.0.0", "request": "GET /images/s102329.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966453, "clientip":"3.0.0.0", "request": "GET /english/images/news_btn_kits_off.gif HTTP/1.0", "status": 200, "size": 933} +{"@timestamp": 893966453, "clientip":"57.0.0.0", "request": "GET /images/ligne01.gif HTTP/1.0", "status": 200, "size": 169} +{"@timestamp": 893966453, "clientip":"7.0.0.0", "request": "GET /english/playing/images/links.GIF HTTP/1.0", "status": 200, "size": 1394} +{"@timestamp": 893966453, "clientip":"47.0.0.0", "request": "GET /images/s102377.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966454, "clientip":"47.0.0.0", "request": "GET /images/s102328.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966454, "clientip":"66.0.0.0", "request": "GET /images/home_fr_button.gif HTTP/1.0", "status": 200, "size": 2140} +{"@timestamp": 893966454, "clientip":"47.0.0.0", "request": "GET /images/s102323.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966454, "clientip":"34.0.0.0", "request": "GET /english/history/images/history_hm_3094.gif HTTP/1.0", "status": 200, "size": 1031} +{"@timestamp": 893966454, "clientip":"69.0.0.0", "request": "GET /english/history/history_of/images/cup/trophytxt.gif HTTP/1.0", "status": 200, "size": 1425} +{"@timestamp": 893966454, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/19historical.gif HTTP/1.0", "status": 200, "size": 616} +{"@timestamp": 893966454, "clientip":"57.0.0.0", "request": "GET /images/base.gif HTTP/1.0", "status": 200, "size": 366} +{"@timestamp": 893966454, "clientip":"79.0.0.0", "request": "GET /english/history/images/history_hm_nav.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966454, "clientip":"47.0.0.0", "request": "GET /english/images/team_group_header_f.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966454, "clientip":"6.0.0.0", "request": "GET /french/venues/cities/images/denis/18collect.gif HTTP/1.0", "status": 200, "size": 714} +{"@timestamp": 893966454, "clientip":"47.0.0.0", "request": "GET /images/s102442.gif HTTP/1.0", "status": 304, "size": 0} +{"@timestamp": 893966455, "clientip":"34.0.0.0", "request": "GET /english/history/images/history_hm_nav.gif HTTP/1.0", "status": 200, "size": 18328} diff --git a/regression-test/data/inverted_index_p0/storage_format/test_storage_format_v1.out b/regression-test/data/inverted_index_p0/storage_format/test_storage_format_v1.out new file mode 100644 index 0000000000..fb40ffbfcb --- /dev/null +++ b/regression-test/data/inverted_index_p0/storage_format/test_storage_format_v1.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +863 + +-- !sql -- +863 863 + +-- !sql -- +863 + +-- !sql -- +827 + +-- !sql -- +844 + diff --git a/regression-test/data/show_p0/test_show_create_table_and_views.out b/regression-test/data/show_p0/test_show_create_table_and_views.out index c6b261e5b6..e97b1864c2 100644 --- a/regression-test/data/show_p0/test_show_create_table_and_views.out +++ b/regression-test/data/show_p0/test_show_create_table_and_views.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !show -- -show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !select -- 1 1 30 @@ -36,11 +36,11 @@ show_create_table_and_views_view CREATE VIEW `show_create_table_and_views_view` 300 1 -- !show -- -show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !show -- -show_create_table_and_views_like CREATE TABLE `show_create_table_and_views_like` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_like CREATE TABLE `show_create_table_and_views_like` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !show -- -show_create_table_and_views_like_with_rollup CREATE TABLE `show_create_table_and_views_like_with_rollup` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_like_with_rollup CREATE TABLE `show_create_table_and_views_like_with_rollup` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); diff --git a/regression-test/suites/fault_injection_p0/test_index_compound_directory_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_index_compound_directory_fault_injection.groovy index dfd056c978..759c409a85 100644 --- a/regression-test/suites/fault_injection_p0/test_index_compound_directory_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_index_compound_directory_fault_injection.groovy @@ -90,67 +90,67 @@ suite("test_index_compound_directory_failure_injection", "nonConcurrent") { create_httplogs_dup_table.call(testTable_dup) try { - GetDebugPoint().enableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._throw_clucene_error_in_fsindexoutput_destructor") + GetDebugPoint().enableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._throw_clucene_error_in_fsindexoutput_destructor") load_httplogs_data.call(testTable_dup, 'test_index_compound_directory', 'true', 'json', 'documents-1000.json') } finally { - GetDebugPoint().disableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._throw_clucene_error_in_fsindexoutput_destructor") + GetDebugPoint().disableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._throw_clucene_error_in_fsindexoutput_destructor") } qt_sql "select COUNT() from ${testTable_dup} where request match 'images'" try { - GetDebugPoint().enableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._throw_clucene_error_in_bufferedindexoutput_close") + GetDebugPoint().enableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._throw_clucene_error_in_bufferedindexoutput_close") load_httplogs_data.call(testTable_dup, 'test_index_compound_directory', 'true', 'json', 'documents-1000.json') } finally { - GetDebugPoint().disableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._throw_clucene_error_in_bufferedindexoutput_close") + GetDebugPoint().disableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._throw_clucene_error_in_bufferedindexoutput_close") } qt_sql "select COUNT() from ${testTable_dup} where request match 'images'" try { - GetDebugPoint().enableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._set_writer_finalize_status_error") + GetDebugPoint().enableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._set_writer_finalize_status_error") load_httplogs_data.call(testTable_dup, 'test_index_compound_directory', 'true', 'json', 'documents-1000.json') } finally { - GetDebugPoint().disableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._set_writer_finalize_status_error") + GetDebugPoint().disableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._set_writer_finalize_status_error") } qt_sql "select COUNT() from ${testTable_dup} where request match 'images'" try { - GetDebugPoint().enableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._set_writer_close_status_error") + GetDebugPoint().enableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._set_writer_close_status_error") load_httplogs_data.call(testTable_dup, 'test_index_compound_directory', 'true', 'json', 'documents-1000.json') } finally { - GetDebugPoint().disableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._set_writer_close_status_error") + GetDebugPoint().disableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._set_writer_close_status_error") } qt_sql "select COUNT() from ${testTable_dup} where request match 'images'" try { def test_index_compound_directory = "test_index_compound_directory1" sql "DROP TABLE IF EXISTS ${test_index_compound_directory}" create_httplogs_dup_table.call(test_index_compound_directory) - GetDebugPoint().enableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._mock_append_data_error_in_fsindexoutput_flushBuffer") + GetDebugPoint().enableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._mock_append_data_error_in_fsindexoutput_flushBuffer") load_httplogs_data.call(test_index_compound_directory, test_index_compound_directory, 'true', 'json', 'documents-1000.json') qt_sql "select COUNT() from ${test_index_compound_directory} where request match 'gif'" try_sql("DROP TABLE IF EXISTS ${test_index_compound_directory}") } catch(Exception ex) { logger.info("_mock_append_data_error_in_fsindexoutput_flushBuffer, result: " + ex) } finally { - GetDebugPoint().disableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._mock_append_data_error_in_fsindexoutput_flushBuffer") + GetDebugPoint().disableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._mock_append_data_error_in_fsindexoutput_flushBuffer") } try { def test_index_compound_directory = "test_index_compound_directory2" sql "DROP TABLE IF EXISTS ${test_index_compound_directory}" create_httplogs_dup_table.call(test_index_compound_directory) - GetDebugPoint().enableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._status_error_in_fsindexoutput_flushBuffer") + GetDebugPoint().enableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._status_error_in_fsindexoutput_flushBuffer") load_httplogs_data.call(test_index_compound_directory, test_index_compound_directory, 'true', 'json', 'documents-1000.json') qt_sql "select COUNT() from ${test_index_compound_directory} where request match 'images'" try_sql("DROP TABLE IF EXISTS ${test_index_compound_directory}") } finally { - GetDebugPoint().disableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._status_error_in_fsindexoutput_flushBuffer") + GetDebugPoint().disableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._status_error_in_fsindexoutput_flushBuffer") } try { def test_index_compound_directory = "test_index_compound_directory3" sql "DROP TABLE IF EXISTS ${test_index_compound_directory}" create_httplogs_dup_table.call(test_index_compound_directory) - GetDebugPoint().enableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._throw_clucene_error_in_fsindexoutput_init") + GetDebugPoint().enableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._throw_clucene_error_in_fsindexoutput_init") load_httplogs_data.call(test_index_compound_directory, test_index_compound_directory, 'true', 'json', 'documents-1000.json') qt_sql "select COUNT() from ${test_index_compound_directory} where request match 'png'" try_sql("DROP TABLE IF EXISTS ${test_index_compound_directory}") } finally { - GetDebugPoint().disableDebugPointForAllBEs("DorisCompoundDirectory::FSIndexOutput._throw_clucene_error_in_fsindexoutput_init") + GetDebugPoint().disableDebugPointForAllBEs("DorisFSDirectory::FSIndexOutput._throw_clucene_error_in_fsindexoutput_init") } } finally { //try_sql("DROP TABLE IF EXISTS ${testTable}") diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_1.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_1.groovy index b938f89874..b9713f016e 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_1.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_1.groovy @@ -103,4 +103,71 @@ suite("test_index_change_1") { qt_select4 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ qt_select5 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ qt_select6 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ + + tableName = "test_index_change_1_v1" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `note` TEXT COMMENT "备注") + DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format" = "V1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a scientist') + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher') + """ + + sql """ INSERT INTO ${tableName} VALUES + (6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer') + """ + + qt_select1_v1 """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex; """ + + // create inverted index + sql """ CREATE INDEX idx_user_id ON ${tableName}(`user_id`) USING INVERTED """ + wait_for_latest_op_on_table_finish(tableName, timeout) + sql """ CREATE INDEX idx_note ON ${tableName}(`note`) USING INVERTED PROPERTIES("parser"="english") """ + wait_for_latest_op_on_table_finish(tableName, timeout) + sql """ CREATE INDEX idx_city ON ${tableName}(`city`) USING INVERTED PROPERTIES("parser"="english") """ + wait_for_latest_op_on_table_finish(tableName, timeout) + + // create bitmap index + sql """ CREATE INDEX idx_sex ON ${tableName}(`sex`) USING BITMAP """ + wait_for_latest_op_on_table_finish(tableName, timeout) + + show_result = sql "show index from ${tableName}" + logger.info("show index from " + tableName + " result: " + show_result) + assertEquals(show_result.size(), 4) + assertEquals(show_result[0][2], "idx_user_id") + assertEquals(show_result[1][2], "idx_note") + assertEquals(show_result[2][2], "idx_city") + assertEquals(show_result[3][2], "idx_sex") + + qt_select2_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' ORDER BY user_id; """ + qt_select3_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and sex = 1 ORDER BY user_id; """ + qt_select4_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ + qt_select5_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ + qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ } diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_2.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_2.groovy index 0d9f96d3d0..5c43bda26b 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_2.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_2.groovy @@ -128,4 +128,74 @@ suite("test_index_change_2") { qt_select9 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ qt_select10 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ qt_select11 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ + + tableName = "test_index_change_2_v1" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `note` TEXT COMMENT "备注", + INDEX idx_user_id (`user_id`) USING INVERTED COMMENT '', + INDEX idx_note (`note`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format" = "V1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a scientist') + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher') + """ + + sql """ INSERT INTO ${tableName} VALUES + (6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer') + """ + + qt_select1_v1 """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex; """ + qt_select2_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' ORDER BY user_id; """ + qt_select3_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and sex = 1 ORDER BY user_id; """ + qt_select4_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ + qt_select5_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ + qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ + + // create inverted index idx_city + sql """ CREATE INDEX idx_city ON ${tableName}(`city`) USING INVERTED PROPERTIES("parser"="english") """ + wait_for_latest_op_on_table_finish(tableName, timeout) + + // drop inverted index idx_user_id, idx_note + sql """ DROP INDEX idx_user_id ON ${tableName} """ + wait_for_latest_op_on_table_finish(tableName, timeout) + sql """ DROP INDEX idx_note ON ${tableName} """ + wait_for_build_index_on_partition_finish(tableName, timeout) + + show_result = sql "show index from ${tableName}" + logger.info("show index from " + tableName + " result: " + show_result) + assertEquals(show_result.size(), 1) + assertEquals(show_result[0][2], "idx_city") + + qt_select7_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' ORDER BY user_id; """ + qt_select8_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and sex = 1 ORDER BY user_id; """ + qt_select9_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ + qt_select10_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ + qt_select11_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ } diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy index 257da84381..f4a4cd76d6 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_3.groovy @@ -131,4 +131,77 @@ suite("test_index_change_3") { qt_select9 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ qt_select10 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ qt_select11 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ + + tableName = "test_index_change_3_v1" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `note` TEXT COMMENT "备注", + INDEX idx_user_id (`user_id`) USING INVERTED COMMENT '', + INDEX idx_note (`note`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format" = "V1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a scientist') + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher') + """ + + sql """ INSERT INTO ${tableName} VALUES + (6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer') + """ + + qt_select1_v1 """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex; """ + qt_select2_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' ORDER BY user_id; """ + qt_select3_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and sex = 1 ORDER BY user_id; """ + qt_select4_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ + qt_select5_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ + qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ + + // create inverted index idx_city + sql """ CREATE INDEX idx_city ON ${tableName}(`city`) USING INVERTED PROPERTIES("parser"="english") """ + wait_for_latest_op_on_table_finish(tableName, timeout) + // build index + sql """ BUILD INDEX idx_city ON ${tableName} """ + wait_for_latest_op_on_table_finish(tableName, timeout) + + // drop inverted index idx_user_id, idx_note + sql """ DROP INDEX idx_city ON ${tableName} """ + wait_for_latest_op_on_table_finish(tableName, timeout) + sql """ DROP INDEX idx_note ON ${tableName} """ + wait_for_build_index_on_partition_finish(tableName, timeout) + + show_result = sql "show index from ${tableName}" + logger.info("show index from " + tableName + " result: " + show_result) + assertEquals(show_result.size(), 1) + assertEquals(show_result[0][2], "idx_user_id") + + qt_select7_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' ORDER BY user_id; """ + qt_select8_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and sex = 1 ORDER BY user_id; """ + qt_select9_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ + qt_select10_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ + qt_select11_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ } diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_4.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_4.groovy index 2de566a777..626f4c7f70 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_4.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_4.groovy @@ -130,4 +130,76 @@ suite("test_index_change_4") { qt_select9 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ qt_select10 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ qt_select11 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ + + tableName = "test_index_change_4_v1" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `note` TEXT COMMENT "备注", + INDEX idx_user_id (`user_id`) USING INVERTED COMMENT '', + INDEX idx_note (`note`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format" = "V1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a scientist') + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher') + """ + + sql """ INSERT INTO ${tableName} VALUES + (6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer') + """ + + qt_select1_v1 """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex; """ + qt_select2_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' ORDER BY user_id; """ + qt_select3_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and sex = 1 ORDER BY user_id; """ + qt_select4_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ + qt_select5_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ + qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ + + // drop inverted index idx_user_id, idx_note + sql """ DROP INDEX idx_user_id ON ${tableName} """ + wait_for_latest_op_on_table_finish(tableName, timeout) + sql """ DROP INDEX idx_note ON ${tableName} """ + wait_for_latest_op_on_table_finish(tableName, timeout) + // create inverted index idx_city + sql """ CREATE INDEX idx_note ON ${tableName}(`note`) USING INVERTED PROPERTIES("parser"="english") """ + wait_for_latest_op_on_table_finish(tableName, timeout) + // build index + sql """ BUILD INDEX idx_note ON ${tableName} """ + wait_for_build_index_on_partition_finish(tableName, timeout) + + show_result = sql "show index from ${tableName}" + logger.info("show index from " + tableName + " result: " + show_result) + assertEquals(show_result.size(), 1) + assertEquals(show_result[0][2], "idx_note") + + qt_select7_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' ORDER BY user_id; """ + qt_select8_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and sex = 1 ORDER BY user_id; """ + qt_select9_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ + qt_select10_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ + qt_select11_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ } diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy index 25533af66e..7b03d8835f 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_5.groovy @@ -106,4 +106,74 @@ suite("test_index_change_5") { qt_select9 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ qt_select10 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ qt_select11 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ + + tableName = "test_index_change_5_v1" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `note` TEXT COMMENT "备注", + INDEX idx_user_id (`user_id`) USING INVERTED COMMENT '', + INDEX idx_note (`note`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format" = "V1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a scientist') + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher') + """ + + sql """ INSERT INTO ${tableName} VALUES + (6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer') + """ + + qt_select1_v1 """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex; """ + qt_select2_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' ORDER BY user_id; """ + qt_select3_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and sex = 1 ORDER BY user_id; """ + qt_select4_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ + qt_select5_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ + qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ + + // drop inverted index idx_user_id, idx_note + sql """ DROP INDEX idx_user_id ON ${tableName} """ + wait_for_latest_op_on_table_finish(tableName, timeout) + sql """ DROP INDEX idx_note ON ${tableName} """ + wait_for_latest_op_on_table_finish(tableName, timeout) + // create bitmap index + sql """ CREATE INDEX idx_sex ON ${tableName}(`sex`) USING BITMAP """ + wait_for_latest_op_on_table_finish(tableName, timeout) + // wait_for_build_index_on_partition_finish(tableName, timeout) + + show_result = sql "show index from ${tableName}" + logger.info("show index from " + tableName + " result: " + show_result) + assertEquals(show_result.size(), 1) + assertEquals(show_result[0][2], "idx_sex") + + qt_select7_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' ORDER BY user_id; """ + qt_select8_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and sex = 1 ORDER BY user_id; """ + qt_select9_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ + qt_select10_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ + qt_select11_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ } diff --git a/regression-test/suites/inverted_index_p0/index_change/test_index_change_6.groovy b/regression-test/suites/inverted_index_p0/index_change/test_index_change_6.groovy index edb300fd68..2a4c3ae8a2 100644 --- a/regression-test/suites/inverted_index_p0/index_change/test_index_change_6.groovy +++ b/regression-test/suites/inverted_index_p0/index_change/test_index_change_6.groovy @@ -136,4 +136,82 @@ suite("test_index_change_6") { qt_select9 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ qt_select10 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ qt_select11 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ + + tableName = "test_index_change_6_v1" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `note` TEXT COMMENT "备注", + INDEX idx_user_id (`user_id`) USING INVERTED COMMENT '', + INDEX idx_note (`note`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + DUPLICATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1", "inverted_index_storage_format" = "V1" ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing China', 10, 1, 'Software Developer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing China', 10, 1, 'Communication Engineer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Shanghai China', 10, 1, 'electrical engineer') + """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-02', 'Beijing China', 10, 0, 'Both a teacher and a scientist') + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-02', 'Shenzhen China', 10, 1, 'teacher') + """ + + sql """ INSERT INTO ${tableName} VALUES + (6, '2017-10-03', 'Hongkong China', 10, 1, 'Architectural designer') + """ + + qt_select1_v1 """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex; """ + qt_select2_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' ORDER BY user_id; """ + qt_select3_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and sex = 1 ORDER BY user_id; """ + qt_select4_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ + qt_select5_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ + qt_select6_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ + + // drop inverted index idx_user_id, idx_note + sql """ DROP INDEX idx_user_id ON ${tableName} """ + wait_for_latest_op_on_table_finish(tableName, timeout) + sql """ DROP INDEX idx_note ON ${tableName} """ + wait_for_latest_op_on_table_finish(tableName, timeout) + + // create inverted index + sql """ CREATE INDEX idx_user_id ON ${tableName}(`user_id`) USING INVERTED """ + wait_for_latest_op_on_table_finish(tableName, timeout) + sql """ CREATE INDEX idx_note ON ${tableName}(`note`) USING INVERTED PROPERTIES("parser" = "english") """ + wait_for_latest_op_on_table_finish(tableName, timeout) + + // build inverted index + sql """ BUILD INDEX idx_user_id ON ${tableName} """ + sql """ BUILD INDEX idx_note ON ${tableName} """ + wait_for_build_index_on_partition_finish(tableName, timeout) + + show_result = sql "show index from ${tableName}" + logger.info("show index from " + tableName + " result: " + show_result) + assertEquals(show_result.size(), 2) + assertEquals(show_result[0][2], "idx_user_id") + assertEquals(show_result[1][2], "idx_note") + + qt_select7_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' ORDER BY user_id; """ + qt_select8_v1 """ SELECT * FROM ${tableName} t WHERE city MATCH 'beijing' and sex = 1 ORDER BY user_id; """ + qt_select9_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer' and sex = 0 ORDER BY user_id; """ + qt_select10_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH_PHRASE 'electrical engineer' ORDER BY user_id; """ + qt_select11_v1 """ SELECT * FROM ${tableName} t WHERE note MATCH 'engineer Developer' AND city match_all 'Shanghai China' ORDER BY user_id; """ } diff --git a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_v1.groovy b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_v1.groovy new file mode 100644 index 0000000000..e30dddf54c --- /dev/null +++ b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_v1.groovy @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_storage_format_v1", "p0") { + // define a sql table + def testTable_dup = "httplogs_dup_v1" + + def create_httplogs_dup_table = {testTablex -> + // multi-line sql + def result = sql """ + CREATE TABLE IF NOT EXISTS ${testTablex} ( + `@timestamp` int(11) NULL, + `clientip` varchar(20) NULL, + `request` text NULL, + `status` int(11) NULL, + `size` int(11) NULL, + INDEX size_idx (`size`) USING INVERTED COMMENT '', + INDEX status_idx (`status`) USING INVERTED COMMENT '', + INDEX clientip_idx (`clientip`) USING INVERTED COMMENT '', + INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser"="english") COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`@timestamp`) + COMMENT 'OLAP' + PARTITION BY RANGE(`@timestamp`) + (PARTITION p181998 VALUES [("-2147483648"), ("894225602")), + PARTITION p191998 VALUES [("894225602"), ("894830402")), + PARTITION p201998 VALUES [("894830402"), ("895435201")), + PARTITION p211998 VALUES [("895435201"), ("896040001")), + PARTITION p221998 VALUES [("896040001"), ("896644801")), + PARTITION p231998 VALUES [("896644801"), ("897249601")), + PARTITION p241998 VALUES [("897249601"), ("897854300")), + PARTITION p251998 VALUES [("897854300"), ("2147483647"))) + DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 12 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "inverted_index_storage_format" = "V2", + "compression" = "ZSTD", + "light_schema_change" = "true", + "disable_auto_compaction" = "false" + ); + """ + } + + def load_httplogs_data = {table_name, label, read_flag, format_flag, file_name, ignore_failure=false, + expected_succ_rows = -1, load_to_single_tablet = 'true' -> + + // load the json data + streamLoad { + table "${table_name}" + + // set http request header params + set 'label', label + "_" + UUID.randomUUID().toString() + set 'read_json_by_line', read_flag + set 'format', format_flag + file file_name // import json file + time 10000 // limit inflight 10s + if (expected_succ_rows >= 0) { + set 'max_filter_ratio', '1' + } + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (ignore_failure && expected_succ_rows < 0) { return } + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + if (expected_succ_rows >= 0) { + assertEquals(json.NumberLoadedRows, expected_succ_rows) + } else { + assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + } + } + + try { + sql "DROP TABLE IF EXISTS ${testTable_dup}" + create_httplogs_dup_table.call(testTable_dup) + load_httplogs_data.call(testTable_dup, 'test_httplogs_load_count_on_index', 'true', 'json', 'documents-1000.json') + sql "sync" + + qt_sql(" select COUNT(*) from ${testTable_dup} where request match 'images' ") + + // case1.2: test multiple count on different columns + qt_sql(" select COUNT(size), COUNT(request) from ${testTable_dup} where request match 'images'; ") + + // case1.3: test count on different column + qt_sql(" select COUNT(size) from ${testTable_dup} where request match 'images'; ") + + qt_sql(" select COUNT() from ${testTable_dup} where size > 100; ") + + qt_sql(" select COUNT() from ${testTable_dup} where status = 200; ") + } finally { + //try_sql("DROP TABLE IF EXISTS ${testTable}") + } +} diff --git a/regression-test/suites/inverted_index_p0/test_show_data.groovy b/regression-test/suites/inverted_index_p0/test_show_data.groovy index 147d011b71..b2f29f0dd3 100644 --- a/regression-test/suites/inverted_index_p0/test_show_data.groovy +++ b/regression-test/suites/inverted_index_p0/test_show_data.groovy @@ -357,4 +357,181 @@ suite("test_show_data_for_bkd", "p0") { } finally { //try_sql("DROP TABLE IF EXISTS ${testTable}") } +} + +suite("test_show_data_multi_add", "p0") { + // define a sql table + def testTableWithoutIndex = "test_show_data_httplogs_multi_add_without_index" + def testTableWithIndex = "test_show_data_httplogs_multi_add_with_index" + def delta_time = 5000 + def timeout = 60000 + String database = context.config.getDbNameByFile(context.file) + + def create_httplogs_table_without_index = {testTablex -> + // multi-line sql + def result = sql """ + CREATE TABLE IF NOT EXISTS ${testTablex} ( + `@timestamp` int(11) NULL, + `clientip` varchar(20) NULL, + `request` text NULL, + `status` int(11) NULL, + `size` int(11) NULL, + ) ENGINE=OLAP + DUPLICATE KEY(`@timestamp`) + DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ); + """ + } + def create_httplogs_table_with_index = {testTablex -> + // multi-line sql + def result = sql """ + CREATE TABLE IF NOT EXISTS ${testTablex} ( + `@timestamp` int(11) NULL, + `clientip` varchar(20) NULL, + `request` text NULL, + `status` int(11) NULL, + `size` int(11) NULL, + INDEX status_idx (`status`) USING INVERTED, + INDEX request_idx (`request`) USING INVERTED + ) ENGINE=OLAP + DUPLICATE KEY(`@timestamp`) + DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ); + """ + } + + def load_httplogs_data = {table_name, label, read_flag, format_flag, file_name, ignore_failure=false, + expected_succ_rows = -1, load_to_single_tablet = 'true' -> + + // load the json data + streamLoad { + table "${table_name}" + + // set http request header params + set 'label', label + "_" + UUID.randomUUID().toString() + set 'read_json_by_line', read_flag + set 'format', format_flag + file file_name // import json file + time 10000 // limit inflight 10s + if (expected_succ_rows >= 0) { + set 'max_filter_ratio', '1' + } + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (ignore_failure && expected_succ_rows < 0) { return } + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + if (expected_succ_rows >= 0) { + assertEquals(json.NumberLoadedRows, expected_succ_rows) + } else { + assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + } + } + + def wait_for_show_data_finish = { table_name, OpTimeout, origin_size -> + def useTime = 0 + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + def result = sql """show data from ${database}.${table_name};""" + if (result.size() > 0) { + logger.info(table_name + " show data, detail: " + result[0].toString()) + def size = result[0][2].replace(" KB", "").toDouble() + if (size != origin_size) { + return size + } + } + useTime = t + Thread.sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_show_data_finish timeout, useTime=${useTime}") + return "wait_timeout" + } + + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> + def useTime = 0 + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + def alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + Thread.sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout, useTime=${useTime}") + } + + def wait_for_last_build_index_on_table_finish = { table_name, OpTimeout -> + def useTime = 0 + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + def alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}" ORDER BY JobId """ + + if (alter_res.size() > 0) { + def last_job_state = alter_res[alter_res.size()-1][7]; + if (last_job_state == "FINISHED" || last_job_state == "CANCELLED") { + logger.info(table_name + " last index job finished, state: " + last_job_state + ", detail: " + alter_res) + return last_job_state; + } + } + useTime = t + Thread.sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout, useTime=${useTime}") + return "wait_timeout" + } + + try { + sql "DROP TABLE IF EXISTS ${testTableWithoutIndex}" + + create_httplogs_table_without_index.call(testTableWithoutIndex) + + load_httplogs_data.call(testTableWithoutIndex, 'test_show_data_httplogs_multi_add_without_index', 'true', 'json', 'documents-1000.json') + + sql "sync" + def no_index_size = wait_for_show_data_finish(testTableWithoutIndex, 300000, 0) + assertTrue(no_index_size != "wait_timeout") + sql """ ALTER TABLE ${testTableWithoutIndex} ADD INDEX idx_status (`status`) USING INVERTED; """ + wait_for_latest_op_on_table_finish(testTableWithoutIndex, timeout) + + // BUILD INDEX and expect state is RUNNING + sql """ BUILD INDEX idx_status ON ${testTableWithoutIndex} """ + def state = wait_for_last_build_index_on_table_finish(testTableWithoutIndex, timeout) + assertEquals(state, "FINISHED") + def with_index_size1 = wait_for_show_data_finish(testTableWithoutIndex, 300000, no_index_size) + assertTrue(with_index_size1 != "wait_timeout") + + sql """ ALTER TABLE ${testTableWithoutIndex} ADD INDEX request_idx (`request`) USING INVERTED; """ + wait_for_latest_op_on_table_finish(testTableWithoutIndex, timeout) + + // BUILD INDEX and expect state is RUNNING + sql """ BUILD INDEX request_idx ON ${testTableWithoutIndex} """ + def state2 = wait_for_last_build_index_on_table_finish(testTableWithoutIndex, timeout) + assertEquals(state2, "FINISHED") + def with_index_size2 = wait_for_show_data_finish(testTableWithoutIndex, 300000, with_index_size1) + assertTrue(with_index_size2 != "wait_timeout") + + sql "DROP TABLE IF EXISTS ${testTableWithIndex}" + create_httplogs_table_with_index.call(testTableWithIndex) + load_httplogs_data.call(testTableWithIndex, 'test_show_data_httplogs_multi_add_with_index', 'true', 'json', 'documents-1000.json') + def another_with_index_size = wait_for_show_data_finish(testTableWithIndex, 300000, 0) + assertEquals(another_with_index_size, with_index_size2) + } finally { + //try_sql("DROP TABLE IF EXISTS ${testTable}") + } } \ No newline at end of file