diff --git a/server/modules/filter/masking/maskingrules.cc b/server/modules/filter/masking/maskingrules.cc index ad868fef6..b1fe81de8 100644 --- a/server/modules/filter/masking/maskingrules.cc +++ b/server/modules/filter/masking/maskingrules.cc @@ -22,6 +22,7 @@ #include #include #include +#include using std::auto_ptr; using std::string; @@ -32,6 +33,8 @@ using maxscale::Closer; namespace { +static const char MASKING_DEFAULT_FILL[] = "X"; + static const char KEY_APPLIES_TO[] = "applies_to"; static const char KEY_COLUMN[] = "column"; static const char KEY_DATABASE[] = "database"; @@ -43,6 +46,7 @@ static const char KEY_TABLE[] = "table"; static const char KEY_VALUE[] = "value"; static const char KEY_WITH[] = "with"; static const char KEY_OBFUSCATE[] = "obfuscate"; +static const char KEY_CAPTURE[] = "capture"; /** * @class AccountVerbatim @@ -363,7 +367,11 @@ bool create_rules_from_array(json_t* pRules, vector >& rules) +bool create_rules_from_root(json_t* pRoot, + vector >& rules) { bool parsed = false; json_t* pRules = json_object_get(pRoot, KEY_RULES); @@ -408,7 +417,9 @@ bool create_rules_from_root(json_t* pRoot, vector } else { - MXS_ERROR("The masking rules object contains a `%s` key, but it is not an array.", KEY_RULES); + MXS_ERROR("The masking rules object contains a `%s` key, " + "but it is not an array.", + KEY_RULES); } } @@ -473,7 +484,7 @@ MaskingRules::CaptureRule::CaptureRule(const std::string& column, const std::string& database, const std::vector& applies_to, const std::vector& exempted, - const std::string& regexp, + pcre2_code* regexp, const std::string& fill) : MaskingRules::Rule::Rule(column, table, database, applies_to, exempted) , m_regexp(regexp) @@ -495,6 +506,7 @@ MaskingRules::ObfuscateRule::~ObfuscateRule() MaskingRules::CaptureRule::~CaptureRule() { + pcre2_code_free(m_regexp); } /** Check the Json array for user rules @@ -529,127 +541,369 @@ static bool validate_user_rules(json_t* pApplies_to, json_t* pExempted) return true; } +static json_t* rule_get_object(json_t* pRule, + const char *rule_type) +{ + json_t *pObj = NULL; + // Check 'rule_type' object + if (!pRule || !(pObj = json_object_get(pRule, rule_type))) + { + MXS_ERROR("A masking rule does not contain the '%s' key.", + rule_type); + return NULL; + } + if (!json_is_object(pObj)) + { + MXS_ERROR("A masking rule contains a '%s' key, " + "but the value is not a valid Json object.", + rule_type); + return NULL; + } + return pObj; +} + +/** + * Checks database, table and column values + * + * @param pColumn The database column + * @param pTable The database table + * @param pDatabase The database name + * + * @return true on success, false otherwise + */ +static bool rule_check_database_options(json_t* pColumn, + json_t* pTable, + json_t* pDatabase) +{ + + // Only column is mandatory; both table and database are optional. + if ((pColumn && json_is_string(pColumn)) && + (!pTable || json_is_string(pTable)) && + (!pDatabase || json_is_string(pDatabase))) + { + return true; + } + else + { + if (!pColumn || !json_is_string(pColumn)) + { + MXS_ERROR("The '%s' object of a masking rule does not have " + "the mandatory '%s' key or it's not a valid Json string.", + KEY_REPLACE, + KEY_COLUMN); + } + else + { + MXS_ERROR("In the '%s' object of a masking rule, the keys " + "'%s' and/or '%s' re not valid Json strings.", + KEY_REPLACE, + KEY_TABLE, + KEY_DATABASE); + } + return false; + } +} + +/** + * Returns a Json objet with the fill value + * + * @param pDoc The Json input object + * + * @return A Json object or NULL + */ +static json_t* rule_get_fill(json_t* pDoc) +{ + json_t* pFill = json_object_get(pDoc, KEY_FILL); + + if (!pFill) + { + // Allowed. Use default value for fill and add it to pWith. + pFill = json_string(MASKING_DEFAULT_FILL); + if (pFill) + { + json_object_set_new(pDoc, KEY_FILL, pFill); + } + else + { + MXS_ERROR("json_string() error, cannot produce" + " a valid '%s' object for rule '%s'.", + KEY_FILL, + KEY_REPLACE); + } + } + + return pFill; +} + +/** + * Perform rule checks for all Rule classes + * + * @param pRule The Json rule + * @param applies_to Account instances corresponding to the + * accounts listed in 'applies_to' in the json file. + * @param exempted Account instances corresponding to the + * accounts listed in 'exempted' in the json file. + * + * @return True on success, false on errors. + */ +static bool rule_run_common_checks(json_t* pRule, + vector >* applies_to, + vector >* exempted) +{ + json_t* pApplies_to = json_object_get(pRule, KEY_APPLIES_TO); + json_t* pExempted = json_object_get(pRule, KEY_EXEMPTED); + + // Check for pApplies_to and pExempted + if (!validate_user_rules(pApplies_to, pExempted)) + { + return false; + } + + // Set the account rules + if (pApplies_to && pExempted && + (!get_accounts(KEY_APPLIES_TO, pApplies_to, *applies_to) || + !get_accounts(KEY_EXEMPTED, pExempted, *exempted))) + { + return false; + } + + return true; +} + +/** + * Returns rule values from a Json rule object + * + * @param pRule The Json rule + * @param column The column value from the json file. + * @param table The table value from the json file. + * @param database The database value from the json file. + * + * @return True on success, false on errors. + */ +static bool rule_get_common_values(json_t* pRule, + std::string* column, + std::string* table, + std::string* database) +{ + // Get database, table && column + json_t* pDatabase = json_object_get(pRule, KEY_DATABASE); + json_t* pTable = json_object_get(pRule, KEY_TABLE); + json_t* pColumn = json_object_get(pRule, KEY_COLUMN); + + // Check column/table/dataase + if (!rule_check_database_options(pColumn, + pTable, + pDatabase)) + { + return false; + } + + // Column exists + column->assign(json_string_value(pColumn)); + + // Check optional table and dbname + if (pTable) + { + table->assign(json_string_value(pTable)); + } + if (pDatabase) + { + database->assign(json_string_value(pDatabase)); + } + + return true; +} + +/** + * Check Json object, run common checks and return rule values + * + * @param pRule The Json rule object + * @param applies_to Account instances corresponding to the + * accounts listed in 'applies_to' in the json file. + * @param exempted Account instances corresponding to the + * accounts listed in 'exempted' in the json file. + * @param column The column value from the json file. + * @param table The table value from the json file. + * @param database The database value from the json file. + * @param rule_type The rule_type (obfuscate or replace) + * + * @return True on success, false on errors + */ +bool rule_get_values(json_t* pRule, + vector >* applies_to, + vector >* exempted, + std::string* column, + std::string* table, + std::string* database, + const char *rule_type) +{ + json_t *pKeyObj; + // Get Key object based on 'rule_type' param + if ((pKeyObj = rule_get_object(pRule, + rule_type)) && + // Run checks on user access + rule_run_common_checks(pRule, + applies_to, + exempted) && + // Extract values from the rule + rule_get_common_values(pKeyObj, + column, + table, + database)) + { + return true; + } + + return false; +} + +/** + * Returns 'capture' regexp & 'fill' value from a 'replace' rule + * + * @param pRule The Json rule doc + * @param pCapture The string buffer for 'capture'value + * @param pFill The string buffer for 'fill' value + * + * @return True on success, false on errors + */ +bool rule_get_capture_fill(json_t* pRule, + std::string *pCapture, + std::string* pFill) +{ + // Get the 'with' key from the rule + json_t* pWith = json_object_get(pRule, KEY_WITH); + if (!pWith || !json_is_object(pWith)) + { + MXS_ERROR("A masking '%s' rule doesn't have a valid '%s' key", + KEY_REPLACE, + KEY_WITH); + return false; + } + + // Get the 'replace' rule object + json_t* pKeyObj; + if (!(pKeyObj = rule_get_object(pRule, KEY_REPLACE))) + { + return false; + } + + // Get fill from 'with' object + json_t* pTheFill = rule_get_fill(pWith); + // Get 'capture' from 'replace' ojbect + json_t* pTheCapture = json_object_get(pKeyObj, KEY_CAPTURE); + + // Check values + if ((!pTheFill || !json_is_string(pTheFill)) || + ((!pTheCapture || !json_is_string(pTheCapture)))) + { + MXS_ERROR("A masking '%s' rule has '%s' and/or '%s' " + "invalid Json strings.", + KEY_REPLACE, + KEY_CAPTURE, + KEY_FILL); + return false; + } + else + { + // Update the string buffers + pFill->assign(json_string_value(pTheFill)); + pCapture->assign(json_string_value(pTheCapture)); + + return true; + } +} + +/** + * Returns 'value' & 'fill' from a 'replace' rule + * + * @param pRule The Json rule doc + * @param pValue The string buffer for 'value' + * @param pFill The string buffer for 'fill' + * + * @return True on success, false on errors + */ +bool rule_get_value_fill(json_t* pRule, + std::string *pValue, + std::string* pFill) +{ + // Get the 'with' key from the rule + json_t* pWith = json_object_get(pRule, KEY_WITH); + if (!pWith || !json_is_object(pWith)) + { + MXS_ERROR("A masking '%s' rule doesn't have a valid '%s' key.", + KEY_REPLACE, + KEY_WITH); + return false; + } + + // Get fill from 'with' object + json_t* pTheFill = rule_get_fill(pWith); + + // Get value from 'with' object + json_t* pTheValue = json_object_get(pWith, KEY_VALUE); + + // Check values + if ((!pTheFill || !json_is_string(pTheFill)) || + (!pTheValue || !json_is_string(pTheValue))) + { + MXS_ERROR("A masking '%s' rule has '%s' and/or '%s' " + "invalid Json strings.", + KEY_REPLACE, + KEY_VALUE, + KEY_FILL); + return false; + } + else + { + // Update the string buffers + pFill->assign(json_string_value(pTheFill)); + pValue->assign(json_string_value(pTheValue)); + return true; + } +} + //static auto_ptr MaskingRules::ReplaceRule::create_from(json_t* pRule) { ss_dassert(json_is_object(pRule)); - auto_ptr sRule; - - json_t* pReplace = json_object_get(pRule, KEY_REPLACE); - json_t* pWith = json_object_get(pRule, KEY_WITH); - json_t* pApplies_to = json_object_get(pRule, KEY_APPLIES_TO); - json_t* pExempted = json_object_get(pRule, KEY_EXEMPTED); - - // Check replace && with - if (pReplace && pWith) - { - const char *err = NULL; - - if (!json_is_object(pReplace)) - { - err = KEY_REPLACE; - } - if (!json_is_object(pWith)) - { - err = KEY_WITH; - } - if (err) - { - MXS_ERROR("A masking rule contains a '%s' key, " - "but the value is not an object.", - err); - return sRule; - } - } - else - { - MXS_ERROR("A masking rule does not contain a '%s' and/or a '%s' key.", - KEY_REPLACE, - KEY_WITH); - return sRule; - } - - // Check for pApplies_to and pExempted - if (!validate_user_rules(pApplies_to, pExempted)) - { - return sRule; - } - + json_t *pReplace; + std::string column, table, database, value, fill; vector > applies_to; vector > exempted; + auto_ptr sRule; - // Set the account rules - if (pApplies_to && pExempted && - (!get_accounts(KEY_APPLIES_TO, pApplies_to, applies_to) || - !get_accounts(KEY_EXEMPTED, pExempted, exempted))) + // Check rule, extract base values + if (rule_get_values(pRule, + &applies_to, + &exempted, + &column, + &table, + &database, + KEY_REPLACE) && + rule_get_value_fill(pRule, &value, &fill)) // get value/fill { - return sRule; - } - - // Get database, table && column - json_t* pDatabase = json_object_get(pReplace, KEY_DATABASE); - json_t* pTable = json_object_get(pReplace, KEY_TABLE); - json_t* pColumn = json_object_get(pReplace, KEY_COLUMN); - - // A column is mandatory; both table and database are optional. - if ((pColumn && json_is_string(pColumn)) && - (!pTable || json_is_string(pTable)) && - (!pDatabase || json_is_string(pDatabase))) - { - json_t* pValue = json_object_get(pWith, KEY_VALUE); - json_t* pFill = json_object_get(pWith, KEY_FILL); - - if (!pFill) + if (!value.empty() && !fill.empty()) { - // Allowed. Use default value for fill and add it to pWith. - pFill = json_string("X"); - if (pFill) - { - json_object_set_new(pWith, KEY_FILL, pFill); - } - else - { - MXS_ERROR("json_string() error, cannot produce a valid rule."); - } + // Apply value/fill: instantiate the ReplaceRule class + sRule = auto_ptr(new MaskingRules::ReplaceRule(column, + table, + database, + applies_to, + exempted, + value, + fill)); } - if (pFill) + else { - if ((!pValue || (json_is_string(pValue) && json_string_length(pValue))) && - (json_is_string(pFill) && json_string_length(pFill))) - { - string column(json_string_value(pColumn)); - string table(pTable ? json_string_value(pTable) : ""); - string database(pDatabase ? json_string_value(pDatabase) : ""); - string value(pValue ? json_string_value(pValue) : ""); - string fill(pFill ? json_string_value(pFill) : ""); - - sRule = auto_ptr(new MaskingRules::ReplaceRule(column, - table, - database, - applies_to, - exempted, - value, - fill)); - } - else - { - MXS_ERROR("One of the keys '%s' or '%s' of masking rule object '%s' " - "has a non-string value or the string is empty.", - KEY_VALUE, KEY_FILL, KEY_WITH); - } + MXS_ERROR("Key '%s' or '%s' of masking '%s' rule object '%s' " + "has a non-string value or empty value.", + KEY_VALUE, + KEY_FILL, + KEY_REPLACE, + KEY_WITH); } } - else - { - MXS_ERROR("The '%s' object of a masking rule does not have a '%s' key, or " - "the values of that key and/or possible '%s' and '%s' keys are " - "not strings.", - KEY_REPLACE, - KEY_COLUMN, - KEY_TABLE, - KEY_DATABASE); - } return sRule; } @@ -659,71 +913,105 @@ auto_ptr MaskingRules::ObfuscateRule::create_from(json_t* pR { ss_dassert(json_is_object(pRule)); - auto_ptr sRule; - - // Get obfuscate - json_t* pObfuscate = json_object_get(pRule, KEY_OBFUSCATE); - // Get applies_to - json_t* pApplies_to = json_object_get(pRule, KEY_APPLIES_TO); - // Get applies_to - json_t* pExempted = json_object_get(pRule, KEY_EXEMPTED); - - // Check the pObfuscate object - if (pObfuscate && !json_is_object(pObfuscate)) - { - MXS_ERROR("A masking rule contains a '%s' key, " - "but the value is not an object.", - KEY_OBFUSCATE); - return sRule; - } - - // Check for pApplies_to and pExempted - if (!validate_user_rules(pApplies_to, pExempted)) - { - return sRule; - } - + std::string column, table, database; vector > applies_to; vector > exempted; + auto_ptr sRule; - // Set the account rules - if (pApplies_to && pExempted && - (!get_accounts(KEY_APPLIES_TO, pApplies_to, applies_to) || - !get_accounts(KEY_EXEMPTED, pExempted, exempted))) + // Check rule, extract base values + if (rule_get_values(pRule, + &applies_to, + &exempted, + &column, + &table, + &database, + KEY_OBFUSCATE)) { - return sRule; - } - - // Get database, table and column from obfuscate object - json_t* pDatabase = json_object_get(pObfuscate, KEY_DATABASE); - json_t* pTable = json_object_get(pObfuscate, KEY_TABLE); - json_t* pColumn = json_object_get(pObfuscate, KEY_COLUMN); - - // A column is mandatory; both table and database are optional. - if ((pColumn && json_is_string(pColumn)) && - (!pTable || json_is_string(pTable)) && - (!pDatabase || json_is_string(pDatabase))) - { - // Instantiate the ObfuscateRule class - string column(json_string_value(pColumn)); - string table(pTable ? json_string_value(pTable) : ""); - string database(pDatabase ? json_string_value(pDatabase) : ""); - sRule = auto_ptr(new MaskingRules::ObfuscateRule(column, table, database, applies_to, exempted)); } - else + + return sRule; +} + +/** + * Compiles a pcre2 pattern match + * + * @param match_string The pattern match to compile + * + * @return A valid pcre2_code code or NULL on errors. + */ +static pcre2_code* rule_compile_pcre2_match(const char* match_string) +{ + int errcode; + PCRE2_SIZE erroffset; + // Compile regexp + pcre2_code* pCode = pcre2_compile((PCRE2_SPTR)match_string, + PCRE2_ZERO_TERMINATED, + 0, + &errcode, + &erroffset, + NULL); + if (!pCode) { - MXS_ERROR("The '%s' object of a masking rule does not have a '%s' key, or " - "the values of that key and/or possible '%s' and '%s' keys are " - "not strings.", - KEY_OBFUSCATE, - KEY_COLUMN, - KEY_TABLE, - KEY_DATABASE); + PCRE2_UCHAR errbuf[512]; + pcre2_get_error_message(errcode, errbuf, sizeof(errbuf)); + MXS_ERROR("Regex compilation failed at %d for regex '%s': %s", + (int)erroffset, match_string, errbuf); + return NULL; + } + + return pCode; +} + +//static +auto_ptr MaskingRules::CaptureRule::create_from(json_t* pRule) +{ + ss_dassert(json_is_object(pRule)); + + std::string column, table, database, value, fill, capture; + vector > applies_to; + vector > exempted; + auto_ptr sRule; + + // Check rule, extract base values + // Note: the capture rule has same rule_type of "replace" + if (rule_get_values(pRule, + &applies_to, + &exempted, + &column, + &table, + &database, + KEY_REPLACE) && + rule_get_capture_fill(pRule, // get capture/fill + &capture, + &fill)) + { + + if (!capture.empty() && !fill.empty()) + { + // Compile the regexp capture + pcre2_code* pCode = rule_compile_pcre2_match(capture.c_str()); + + if (pCode) + { + Closer code(pCode); + // Instantiate the CaptureRule class + sRule = auto_ptr(new MaskingRules::CaptureRule(column, + table, + database, + applies_to, + exempted, + pCode, + fill)); + + // Ownership of pCode has been moved to the CaptureRule object. + code.release(); + } + } } return sRule; @@ -834,8 +1122,91 @@ static inline char maxscale_basic_obfuscation(const char c) return c; } +/** + * Fills a buffer with a fill string + * + * @param f_first The iterator pointing to first fill byt + * @param f_last The iterator pointing to last fill byte + * @param o_first The iterator pointing to first buffer byte + * @param o_last The iterator pointing to last buffer byte + */ +template +inline void fill_buffer(FillIter f_first, + FillIter f_last, + OutIter o_first, + OutIter o_last) +{ + FillIter pFill = f_first; + while (o_first != o_last) + { + *o_first++ = *pFill++; + if (pFill == f_last) + { + pFill = f_first; + } + } +} + void MaskingRules::CaptureRule::rewrite(LEncString& s) const { + int rv = 0; + uint32_t n_matches = 0; + PCRE2_SIZE* ovector = NULL; + // Create the match data object from m_regexp class member + pcre2_match_data* pData = pcre2_match_data_create_from_pattern(m_regexp, NULL); + // Set initial offset to the input beginning + PCRE2_SIZE startoffset = 0; + // Get input string size + size_t total_len = s.length(); + + if (pData) + { + // Get the fill size + size_t fill_len = m_fill.length(); + Closer data(pData); + + // Match all the compiled pattern + while ((startoffset < total_len) && + (rv = pcre2_match(m_regexp, + (PCRE2_SPTR)s.to_string().c_str(), + PCRE2_ZERO_TERMINATED, + startoffset, + 0, + pData, + NULL)) >= 0) + { + // Get offset array value pairs of substrings: $0=0,1 ; $1=2,3 + PCRE2_SIZE* ovector = pcre2_get_ovector_pointer(pData); + + // Get Full Match substring size: $0 is [0] and [1] + size_t substring_len = ovector[1] - ovector[0]; + // Go to Full Match substring offset: 0 + LEncString::iterator i = s.begin() + ovector[0]; + + // Avoid infinite loop in pcre2_match for a zero-length match + if (ovector[1] == ovector[0]) + { + break; + } + + // Copy the fill string into substring + fill_buffer(m_fill.begin(), m_fill.end(), i, i + substring_len); + + // Set offset to the end of Full Match substring or break + startoffset = ovector[1]; + } + + // Log errors, exclding NO_MATCH or PARTIAL + if (rv < 0 && (rv != PCRE2_ERROR_NOMATCH || PCRE2_ERROR_PARTIAL)) + { + MXS_PCRE2_PRINT_ERROR(rv); + } + } + else + { + MXS_ERROR("Allocation of matching data for PCRE2 failed." + " This is most likely caused by a lack of memory"); + } } void MaskingRules::ObfuscateRule::rewrite(LEncString& s) const @@ -869,18 +1240,8 @@ void MaskingRules::ReplaceRule::rewrite(LEncString& s) const LEncString::iterator i = s.begin(); size_t len = m_fill.length(); - while (total_len) - { - if (total_len < len) - { - len = total_len; - } - - std::copy(m_fill.data(), m_fill.data() + len, i); - - i += len; - total_len -= len; - } + // Copy the fill string + fill_buffer(m_fill.begin(), m_fill.end(), s.begin(), s.end()); } else { diff --git a/server/modules/filter/masking/maskingrules.hh b/server/modules/filter/masking/maskingrules.hh index e27386f05..2a09f8c82 100644 --- a/server/modules/filter/masking/maskingrules.hh +++ b/server/modules/filter/masking/maskingrules.hh @@ -256,7 +256,7 @@ public: * accounts listed in 'applies_to' in the json file. * @param exempted Account instances corresponding to the * accounts listed in 'exempted' in the json file. - * @param regexp The capture regexp from the json file. + * @param regexp The compiled capture regexp from the json file. * @param fill The fill value from the json file. */ CaptureRule(const std::string& column, @@ -264,14 +264,14 @@ public: const std::string& database, const std::vector& applies_to, const std::vector& exempted, - const std::string& regexp, + pcre2_code* regexp, const std::string& fill); ~CaptureRule(); - const std::string& capture() const + const pcre2_code& capture() const { - return m_regexp; + return *m_regexp; } const std::string& fill() const @@ -297,7 +297,7 @@ public: void rewrite(LEncString& s) const; private: - std::string m_regexp; + pcre2_code* m_regexp; std::string m_fill; private: diff --git a/server/modules/filter/masking/mysql.hh b/server/modules/filter/masking/mysql.hh index 44b5b65a5..3f9b63e11 100644 --- a/server/modules/filter/masking/mysql.hh +++ b/server/modules/filter/masking/mysql.hh @@ -140,6 +140,14 @@ public: return rv; } + iterator operator + (ptrdiff_t n) + { + ss_dassert(m_pS); + iterator rv = m_pS; + rv += n; + return rv; + } + iterator& operator += (ptrdiff_t n) { ss_dassert(m_pS);