Improved support for non-ascii symbol names

lib/libclangmm

@@ -1 +1 @@
-Subproject commit c807211edcd3894f0920fcc1c01476d898f93f8f
+Subproject commit fc0df2022518af9a04b28e4964fb84ace5b1fe81

src/autocomplete.cpp

@@ -65,8 +65,7 @@ void Autocomplete::run() {
     if(pass_buffer_and_strip_word) {
       auto pos = iter.get_offset() - 1;
       buffer = view->get_buffer()->get_text();
-      while(pos >= 0 && ((buffer[pos] >= 'a' && buffer[pos] <= 'z') || (buffer[pos] >= 'A' && buffer[pos] <= 'Z') ||
+      while(pos >= 0 && Source::BaseView::is_token_char(buffer[pos])) {
-                         (buffer[pos] >= '0' && buffer[pos] <= '9') || buffer[pos] == '_')) {
         buffer.replace(pos, 1, " ");
         column_nr--;
         pos--;

src/autocomplete.hpp

@@ -39,7 +39,7 @@ public:
   std::function<std::unique_ptr<LockGuard>()> get_parse_lock = [] { return nullptr; };
   std::function<void()> stop_parse = [] {};
 
-  std::function<bool(guint last_keyval)> is_continue_key = [](guint keyval) { return (keyval >= '0' && keyval <= '9') || (keyval >= 'a' && keyval <= 'z') || (keyval >= 'A' && keyval <= 'Z') || keyval == '_' || gdk_keyval_to_unicode(keyval) >= 0x00C0; };
+  std::function<bool(guint last_keyval)> is_continue_key = [](guint keyval) { return Source::BaseView::is_token_char(gdk_keyval_to_unicode(keyval)); };
   std::function<bool(guint last_keyval)> is_restart_key = [](guint) { return false; };
   std::function<bool()> run_check = [] { return false; };
 

src/ctags.cpp

@@ -64,7 +64,7 @@ Ctags::Location Ctags::get_location(const std::string &line_, bool add_markup, b
   location.symbol = line.substr(0, symbol_end);
   if(9 < location.symbol.size() && location.symbol[8] == ' ' && starts_with(location.symbol, "operator")) {
     auto &chr = location.symbol[9];
-    if(!((chr >= 'a' && chr <= 'z') || (chr >= 'A' && chr <= 'Z') || (chr >= '0' && chr <= '9') || chr == '_'))
+    if(!Source::BaseView::is_token_char(chr))
       location.symbol.erase(8, 1);
   }
 
@@ -174,13 +174,13 @@ Ctags::Location Ctags::get_location(const std::string &line_, bool add_markup, b
   return location;
 }
 
-///Split up a type into its various significant parts
+// Split up a type into its various significant parts
 std::vector<std::string> Ctags::get_type_parts(const std::string &type) {
   std::vector<std::string> parts;
   size_t text_start = std::string::npos;
   for(size_t c = 0; c < type.size(); ++c) {
     auto &chr = type[c];
-    if((chr >= '0' && chr <= '9') || (chr >= 'a' && chr <= 'z') || (chr >= 'A' && chr <= 'Z') || chr == '_' || chr == '~') {
+    if(Source::BaseView::is_token_char(chr) || chr == '~') {
       if(text_start == std::string::npos)
         text_start = c;
     }

src/debug_lldb.cpp

@@ -63,7 +63,7 @@ std::tuple<std::vector<std::string>, std::string, std::vector<std::string>> Debu
       if(c > 0 && start_pos != std::string::npos) {
         auto argument = command.substr(start_pos, c - start_pos);
         if(executable.empty()) {
-          //Check for environment variable
+          // Check for environment variable
           bool env_arg = false;
           for(size_t c = 0; c < argument.size(); ++c) {
             if((argument[c] >= 'a' && argument[c] <= 'z') || (argument[c] >= 'A' && argument[c] <= 'Z') ||

src/selection_dialog.cpp

@@ -412,7 +412,7 @@ bool CompletionDialog::on_key_release(GdkEventKey *event) {
 }
 
 bool CompletionDialog::on_key_press(GdkEventKey *event) {
-  if((event->keyval >= '0' && event->keyval <= '9') || (event->keyval >= 'a' && event->keyval <= 'z') || (event->keyval >= 'A' && event->keyval <= 'Z') || event->keyval == '_' || gdk_keyval_to_unicode(event->keyval) >= 0x00C0 || event->keyval == GDK_KEY_BackSpace) {
+  if(Source::BaseView::is_token_char(gdk_keyval_to_unicode(event->keyval)) || event->keyval == GDK_KEY_BackSpace) {
     if(row_in_entry) {
       text_view->get_buffer()->erase(start_mark->get_iter(), text_view->get_buffer()->get_insert()->get_iter());
       row_in_entry = false;

src/source.cpp

@@ -1011,7 +1011,7 @@ void Source::View::extend_selection() {
 
   // It is impossible to identify <> used for templates by syntax alone, but
   // this function works in most cases.
-  auto is_template_arguments = [this](Gtk::TextIter start, Gtk::TextIter end) {
+  auto is_template_arguments = [](Gtk::TextIter start, Gtk::TextIter end) {
     if(*start != '<' || *end != '>' || start.get_line() != end.get_line())
       return false;
     auto prev = start;
@@ -1602,7 +1602,7 @@ void Source::View::show_or_hide() {
               else if(std::none_of(exact.begin(), exact.end(), [&text](const std::string &e) {
                         return starts_with(text, e);
                       }) &&
-                      std::none_of(followed_by_non_token_char.begin(), followed_by_non_token_char.end(), [this, &text](const std::string &e) {
+                      std::none_of(followed_by_non_token_char.begin(), followed_by_non_token_char.end(), [&text](const std::string &e) {
                         return starts_with(text, e) && text.size() > e.size() && !is_token_char(text[e.size()]);
                       })) {
                 end = get_buffer()->get_iter_at_line(end.get_line());
@@ -2622,7 +2622,7 @@ bool Source::View::on_key_press_event_bracket_language(GdkEventKey *event) {
               if(it == start_iter)
                 break;
               if(!square_outside_para_found && square_count == 0 && para_count == 0) {
-                if((*it >= 'A' && *it <= 'Z') || (*it >= 'a' && *it <= 'z') || (*it >= '0' && *it <= '9') || *it == '_' ||
+                if(is_token_char(*it) ||
                    *it == '-' || *it == ' ' || *it == '\t' || *it == '<' || *it == '>' || *it == '(' || *it == ':' ||
                    *it == '*' || *it == '&' || *it == '/' || it.ends_line() || !is_code_iter(it)) {
                   continue;

src/source_base.cpp

@@ -701,9 +701,7 @@ Gtk::TextIter Source::BaseView::get_tabs_end_iter() {
 }
 
 bool Source::BaseView::is_token_char(gunichar chr) {
-  if((chr >= 'A' && chr <= 'Z') || (chr >= 'a' && chr <= 'z') || (chr >= '0' && chr <= '9') || chr == '_' || chr >= 128)
+  return (chr >= 'A' && chr <= 'Z') || (chr >= 'a' && chr <= 'z') || (chr >= '0' && chr <= '9') || chr == '_' || chr == '$' || chr >= 128;
-    return true;
-  return false;
 }
 
 std::pair<Gtk::TextIter, Gtk::TextIter> Source::BaseView::get_token_iters(Gtk::TextIter iter) {
@@ -1424,63 +1422,70 @@ void Source::BaseView::insert_snippet(Gtk::TextIter iter, const std::string &sni
     }
     return false;
   };
-  auto parse_variable = [&] {
+
+  enum class ParseVariableResult {
+    parsed,
+    skipped,
+    not_found
+  };
+
+  auto parse_variable = [&]() -> ParseVariableResult {
     if(i >= snippet.size())
       throw std::out_of_range("unexpected end");
     if(compare_variable("TM_SELECTED_TEXT")) {
       Gtk::TextIter start, end;
       if(get_buffer()->get_selection_bounds(start, end)) {
         insert += get_buffer()->get_text(start, end);
-        return true;
+        return ParseVariableResult::parsed;
       }
-      return false;
+      return ParseVariableResult::skipped;
     }
     else if(compare_variable("TM_CURRENT_LINE")) {
       auto start = get_buffer()->get_iter_at_line(iter.get_line());
       auto end = get_iter_at_line_end(iter.get_line());
       insert += get_buffer()->get_text(start, end);
       erase_line = true;
-      return true;
+      return ParseVariableResult::parsed;
     }
     else if(compare_variable("TM_CURRENT_WORD")) {
       if(is_token_char(*iter)) {
         auto token_iters = get_token_iters(iter);
         insert += get_buffer()->get_text(token_iters.first, token_iters.second);
         erase_word = true;
-        return true;
+        return ParseVariableResult::parsed;
       }
-      return false;
+      return ParseVariableResult::skipped;
     }
     else if(compare_variable("TM_LINE_INDEX")) {
       insert += std::to_string(iter.get_line());
-      return true;
+      return ParseVariableResult::parsed;
     }
     else if(compare_variable("TM_LINE_NUMBER")) {
       insert += std::to_string(iter.get_line() + 1);
-      return true;
+      return ParseVariableResult::parsed;
     }
     else if(compare_variable("TM_FILENAME_BASE")) {
       insert += file_path.stem().string();
-      return true;
+      return ParseVariableResult::parsed;
     }
     else if(compare_variable("TM_FILENAME")) {
       insert += file_path.filename().string();
-      return true;
+      return ParseVariableResult::parsed;
     }
     else if(compare_variable("TM_DIRECTORY")) {
       insert += file_path.parent_path().string();
-      return true;
+      return ParseVariableResult::parsed;
     }
     else if(compare_variable("TM_FILEPATH")) {
       insert += file_path.string();
-      return true;
+      return ParseVariableResult::parsed;
     }
     else if(compare_variable("CLIPBOARD")) {
       insert += Gtk::Clipboard::get()->wait_for_text();
-      return true;
+      return ParseVariableResult::parsed;
     }
     // TODO: support other variables
-    return false;
+    return ParseVariableResult::not_found;
   };
 
   std::function<void(bool)> parse_snippet = [&](bool stop_at_curly_end) {
@@ -1517,10 +1522,10 @@ void Source::BaseView::insert_snippet(Gtk::TextIter iter, const std::string &sni
             parameter_offsets_and_sizes_map[number].emplace_back(utf8_character_count(insert) - placeholder_character_count, placeholder_character_count);
           }
           else {
-            if(!parse_variable()) {
+            if(parse_variable() != ParseVariableResult::parsed) {
               if(snippet.at(i) == ':') { // Use default value
                 ++i;
-                if(!parse_variable())
+                if(parse_variable() != ParseVariableResult::parsed)
                   parse_snippet(true);
               }
             }
@@ -1535,8 +1540,8 @@ void Source::BaseView::insert_snippet(Gtk::TextIter iter, const std::string &sni
         }
         else if(parse_number(number))
           parameter_offsets_and_sizes_map[number].emplace_back(utf8_character_count(insert), 0);
-        else
+        else if(parse_variable() == ParseVariableResult::not_found)
-          parse_variable();
+          insert += '$';
       }
       else {
         insert += snippet[i];

src/source_base.hpp

@@ -153,7 +153,10 @@ namespace Source {
     Gtk::TextIter get_tabs_end_iter(int line_nr);
     Gtk::TextIter get_tabs_end_iter();
 
-    bool is_token_char(gunichar chr);
+  public:
+    static bool is_token_char(gunichar chr);
+
+  protected:
     std::pair<Gtk::TextIter, Gtk::TextIter> get_token_iters(Gtk::TextIter iter);
     std::string get_token(const Gtk::TextIter &iter);
     void cleanup_whitespace_characters();

src/source_clang.cpp

@@ -857,6 +857,7 @@ Source::ClangViewAutocomplete::ClangViewAutocomplete(const boost::filesystem::pa
 
   autocomplete.run_check = [this]() {
     auto iter = get_buffer()->get_insert()->get_iter();
+    auto prefix_end = iter;
     iter.backward_char();
     if(!is_code_iter(iter))
       return false;
@@ -864,31 +865,57 @@ Source::ClangViewAutocomplete::ClangViewAutocomplete(const boost::filesystem::pa
     enable_snippets = false;
     show_parameters = false;
 
-    auto line = ' ' + get_line_before();
+    size_t count = 0;
-    const static std::regex regex("^.*([a-zA-Z_\\)\\]\\>]|[^a-zA-Z0-9_][a-zA-Z_][a-zA-Z0-9_]*)(\\.|->)([a-zA-Z0-9_]*)$|" // . or ->
+    while(is_token_char(*iter) && iter.backward_char())
-                                  "^.*(::)([a-zA-Z0-9_]*)$|"                                                             // ::
+      ++count;
-                                  "^.*[^a-zA-Z0-9_]([a-zA-Z_][a-zA-Z0-9_]{2,})$",                                        // part of symbol
+
-                                  std::regex::optimize);
+    auto prefix_start = iter;
-    std::smatch sm;
+    if(prefix_start != prefix_end)
-    if(std::regex_match(line, sm, regex)) {
+      prefix_start.forward_char();
+
+    auto previous = iter;
+    if(*iter == '.') {
+      bool starts_with_num = false;
+      size_t count = 0;
+      while(iter.backward_char() && is_token_char(*iter)) {
+        ++count;
+        starts_with_num = Glib::Unicode::isdigit(*iter);
+      }
+      if((count >= 1 || *iter == ')' || *iter == ']') && !starts_with_num) {
         {
           LockGuard lock(autocomplete.prefix_mutex);
-        autocomplete.prefix = sm.length(2) ? sm[3].str() : sm.length(4) ? sm[5].str() : sm[6].str();
+          autocomplete.prefix = get_buffer()->get_text(prefix_start, prefix_end);
-        if(!sm.length(2) && !sm.length(4))
+        }
-          enable_snippets = true;
+        return true;
+      }
+    }
+    else if((previous.backward_char() && ((*previous == ':' && *iter == ':') || (*previous == '-' && *iter == '>')))) {
+      if(*iter == ':' || (previous.backward_char() && (is_token_char(*previous) || *previous == ')' || *previous == ']'))) {
+        {
+          LockGuard lock(autocomplete.prefix_mutex);
+          autocomplete.prefix = get_buffer()->get_text(prefix_start, prefix_end);
+        }
+        return true;
+      }
+    }
+    else if(count >= 3) { // part of symbol
+      {
+        LockGuard lock(autocomplete.prefix_mutex);
+        autocomplete.prefix = get_buffer()->get_text(prefix_start, prefix_end);
       }
+      enable_snippets = true;
       return true;
     }
-    else if(is_possible_argument()) {
+    if(is_possible_argument()) {
       show_parameters = true;
       LockGuard lock(autocomplete.prefix_mutex);
       autocomplete.prefix = "";
       return true;
     }
-    else if(!interactive_completion) {
+    if(!interactive_completion) {
       auto end_iter = get_buffer()->get_insert()->get_iter();
       auto iter = end_iter;
-      while(iter.backward_char() && autocomplete.is_continue_key(*iter)) {
+      while(iter.backward_char() && is_token_char(*iter)) {
       }
       if(iter != end_iter)
         iter.forward_char();
@@ -1707,11 +1734,7 @@ Source::ClangViewRefactor::ClangViewRefactor(const boost::filesystem::path &file
       if(token.is_identifier()) {
         auto &token_offsets = clang_tokens_offsets[c];
         if(line == token_offsets.first.line - 1 && index >= token_offsets.first.index - 1 && index <= token_offsets.second.index - 1) {
-          auto token_spelling = token.get_spelling();
+          if(get_token(iter) == token.get_spelling()) {
-          if(!token_spelling.empty() &&
-             (token_spelling.size() > 1 || (token_spelling.back() >= 'a' && token_spelling.back() <= 'z') ||
-              (token_spelling.back() >= 'A' && token_spelling.back() <= 'Z') ||
-              token_spelling.back() == '_')) {
             auto cursor = token.get_cursor();
             auto kind = cursor.get_kind();
             if(kind == clangmm::Cursor::Kind::FunctionDecl || kind == clangmm::Cursor::Kind::CXXMethod ||

src/source_generic.cpp

@@ -203,15 +203,11 @@ void Source::GenericView::setup_autocomplete() {
     autocomplete_insert.clear();
   };
 
-  autocomplete.is_restart_key = [](guint keyval) {
-    return false;
-  };
-
   autocomplete.run_check = [this]() {
     auto start = get_buffer()->get_insert()->get_iter();
     auto end = start;
     size_t count = 0;
-    while(start.backward_char() && ((*start >= '0' && *start <= '9') || (*start >= 'a' && *start <= 'z') || (*start >= 'A' && *start <= 'Z') || *start == '_' || *start >= 0x00C0))
+    while(start.backward_char() && is_token_char(*start))
       ++count;
     if((start.is_start() || start.forward_char()) && count >= 3 && !(*start >= '0' && *start <= '9')) {
       LockGuard lock1(autocomplete.prefix_mutex);
@@ -223,7 +219,7 @@ void Source::GenericView::setup_autocomplete() {
     else if(!interactive_completion) {
       auto end_iter = get_buffer()->get_insert()->get_iter();
       auto iter = end_iter;
-      while(iter.backward_char() && autocomplete.is_continue_key(*iter)) {
+      while(iter.backward_char() && is_token_char(*iter)) {
       }
       if(iter != end_iter)
         iter.forward_char();

src/source_language_protocol.cpp

@@ -1531,6 +1531,7 @@ void Source::LanguageProtocolView::setup_autocomplete() {
 
   autocomplete->run_check = [this, is_possible_jsx_property]() {
     auto iter = get_buffer()->get_insert()->get_iter();
+    auto prefix_end = iter;
     iter.backward_char();
     if(!is_code_iter(iter))
       return false;
@@ -1538,36 +1539,60 @@ void Source::LanguageProtocolView::setup_autocomplete() {
     autocomplete_enable_snippets = false;
     autocomplete_show_arguments = false;
 
-    auto line = ' ' + get_line_before();
+    size_t count = 0;
-    const static std::regex regex("^.*([a-zA-Z_\\)\\]\\>\"']|[^a-zA-Z0-9_][a-zA-Z_][a-zA-Z0-9_]*\\?{0,1})(\\.)([a-zA-Z0-9_]*)$|" // .
+    while(is_token_char(*iter) && iter.backward_char())
-                                  "^.*(::)([a-zA-Z0-9_]*)$|"                                                                     // ::
+      ++count;
-                                  "^.*[^a-zA-Z0-9_]([a-zA-Z_][a-zA-Z0-9_]{2,})$",                                                // part of symbol
+
-                                  std::regex::optimize);
+    auto prefix_start = iter;
-    std::smatch sm;
+    if(prefix_start != prefix_end)
-    if(std::regex_match(line, sm, regex)) {
+      prefix_start.forward_char();
+
+    auto previous = iter;
+    if(*iter == '.') {
+      bool starts_with_num = false;
+      size_t count = 0;
+      while(iter.backward_char() && is_token_char(*iter)) {
+        ++count;
+        starts_with_num = Glib::Unicode::isdigit(*iter);
+      }
+      if((count >= 1 || *iter == ')' || *iter == ']' || *iter == '"' || *iter == '\'' || *iter == '?') && !starts_with_num) {
         {
           LockGuard lock(autocomplete->prefix_mutex);
-        autocomplete->prefix = sm.length(2) ? sm[3].str() : sm.length(4) ? sm[5].str() : sm[6].str();
+          autocomplete->prefix = get_buffer()->get_text(prefix_start, prefix_end);
-        if(!sm.length(2) && !sm.length(4))
+        }
-          autocomplete_enable_snippets = true;
+        return true;
+      }
+    }
+    else if((previous.backward_char() && *previous == ':' && *iter == ':')) {
+      {
+        LockGuard lock(autocomplete->prefix_mutex);
+        autocomplete->prefix = get_buffer()->get_text(prefix_start, prefix_end);
       }
       return true;
     }
-    else if(is_possible_jsx_property(iter)) {
+    else if(count >= 3) { // part of symbol
+      {
+        LockGuard lock(autocomplete->prefix_mutex);
+        autocomplete->prefix = get_buffer()->get_text(prefix_start, prefix_end);
+      }
+      autocomplete_enable_snippets = true;
+      return true;
+    }
+    if(is_possible_jsx_property(iter)) {
       LockGuard lock(autocomplete->prefix_mutex);
       autocomplete->prefix = "";
       return true;
     }
-    else if(is_possible_argument()) {
+    if(is_possible_argument()) {
       autocomplete_show_arguments = true;
       LockGuard lock(autocomplete->prefix_mutex);
       autocomplete->prefix = "";
       return true;
     }
-    else if(!interactive_completion) {
+    if(!interactive_completion) {
       auto end_iter = get_buffer()->get_insert()->get_iter();
       auto iter = end_iter;
-      while(iter.backward_char() && autocomplete->is_continue_key(*iter)) {
+      while(iter.backward_char() && is_token_char(*iter)) {
       }
       if(iter != end_iter)
         iter.forward_char();

src/source_spellcheck.cpp

@@ -495,9 +495,7 @@ bool Source::SpellCheckView::is_word_iter(const Gtk::TextIter &iter) {
     ++backslash_count;
   if(backslash_count % 2 == 1)
     return false;
-  if(*iter >= 0x2030) // Symbols and emojis
+  if(Glib::Unicode::isalpha(*iter))
-    return false;
-  if(((*iter >= 'A' && *iter <= 'Z') || (*iter >= 'a' && *iter <= 'z') || *iter >= 128))
     return true;
   if(*iter == '\'')
     return !is_code_iter(iter);

src/usages_clang.cpp

@@ -588,7 +588,7 @@ std::pair<std::map<boost::filesystem::path, Usages::Clang::PathSet>, Usages::Cla
   const static std::regex include_regex(R"R(^#[ \t]*include[ \t]*"([^"]+)".*$)R", std::regex::optimize);
 
   auto is_spelling_char = [](char chr) {
-    return (chr >= 'a' && chr <= 'z') || (chr >= 'A' && chr <= 'Z') || (chr >= '0' && chr <= '9') || chr == '_';
+    return (chr >= 'a' && chr <= 'z') || (chr >= 'A' && chr <= 'Z') || (chr >= '0' && chr <= '9') || chr == '_' || chr == '$' || static_cast<unsigned char>(chr) >= 128;
   };
 
   for(auto &path : paths) {

tests/source_test.cpp

@@ -677,6 +677,18 @@ int main() {
       g_assert(buffer->get_insert()->get_iter().get_offset() == 13);
     }
     {
+      buffer->set_text("");
+      view.insert_snippet(buffer->get_insert()->get_iter(), "test$test");
+      g_assert(buffer->get_text() == "test$test");
+
+      buffer->set_text("");
+      view.insert_snippet(buffer->get_insert()->get_iter(), "${TM_SELECTED_TEXT}");
+      g_assert(buffer->get_text() == "");
+
+      buffer->set_text("");
+      view.insert_snippet(buffer->get_insert()->get_iter(), "$TM_SELECTED_TEXT");
+      g_assert(buffer->get_text() == "");
+
       buffer->set_text("");
       view.insert_snippet(buffer->get_insert()->get_iter(), "\\$");
       g_assert(buffer->get_text() == "$");