Optimization of Tokens::Tokens, and some minor cleanup

src/SourceLocation.h

@@ -6,8 +6,9 @@
 namespace clangmm {
   class Offset {
   public:
-    bool operator==(const clangmm::Offset &o) const {return line==o.line && index==o.index;}
+    bool operator==(const Offset &o) const {return line==o.line && index==o.index;}
-    bool operator!=(const clangmm::Offset &o) const {return !(*this==o);}
+    bool operator!=(const Offset &o) const {return !(*this==o);}
+    bool operator<(const Offset &o) const {return line<o.line || (line==o.line && index<o.index);}
     unsigned line;
     unsigned index; //byte index in line (not char number)
   };

src/SourceRange.cc

@@ -4,7 +4,15 @@ clangmm::SourceRange::SourceRange(clangmm::SourceLocation &start, clangmm::Sourc
   cx_range = clang_getRange(start.cx_location, end.cx_location);
 }
 
-std::pair<clangmm::Offset, clangmm::Offset> clangmm::SourceRange::get_offsets() {
+clangmm::SourceLocation clangmm::SourceRange::get_start() const {
-  SourceLocation start(clang_getRangeStart(cx_range)), end(clang_getRangeEnd(cx_range));
+  return SourceLocation(clang_getRangeStart(cx_range));
-  return {start.get_offset(), end.get_offset()};
+}
+
+clangmm::SourceLocation clangmm::SourceRange::get_end() const {
+  return SourceLocation(clang_getRangeEnd(cx_range));
+}
+
+
+std::pair<clangmm::Offset, clangmm::Offset> clangmm::SourceRange::get_offsets() const {
+  return {get_start().get_offset(), get_end().get_offset()};
 }

src/SourceRange.h

@@ -10,7 +10,9 @@ namespace clangmm {
   public:
     SourceRange(const CXSourceRange& cx_range) : cx_range(cx_range) {}
     SourceRange(SourceLocation &start, SourceLocation &end);
-    std::pair<clangmm::Offset, clangmm::Offset> get_offsets();
+    SourceLocation get_start() const;
+    SourceLocation get_end() const;
+    std::pair<clangmm::Offset, clangmm::Offset> get_offsets() const;
     CXSourceRange cx_range;
   };
 }  // namespace clangmm

src/Tokens.cc

@@ -2,51 +2,70 @@
 #include "Utility.h"
 #include <unordered_set>
 #include <cstring>
+#include <map>
 
 clangmm::Tokens::Tokens(CXTranslationUnit &cx_tu, const SourceRange &range): cx_tu(cx_tu) {
   unsigned num_tokens;
   clang_tokenize(cx_tu, range.cx_range, &cx_tokens, &num_tokens);
   cx_cursors=std::unique_ptr<CXCursor[]>(new CXCursor[num_tokens]); // To avoid allocation with initialization
   clang_annotateTokens(cx_tu, cx_tokens, num_tokens, cx_cursors.get());
+  
+  bool tu_cursors=SourceRange(clang_getCursorExtent(clang_getTranslationUnitCursor(cx_tu))).get_start().get_path()==range.get_start().get_path();
+  std::map<size_t, Offset> invalid_tokens;
+  
   for (unsigned i = 0; i < num_tokens; i++) {
     if(cx_cursors[i].kind==CXCursor_DeclRefExpr) { // Temporary fix to a libclang bug
       auto real_cursor=clang_getCursor(cx_tu, clang_getTokenLocation(cx_tu, cx_tokens[i]));
       cx_cursors[i]=real_cursor;
     }
     // Corrects: when getting tokens from a header, FieldDecl tokens are getting ClassDecl or StructDecl cursors
-    else if(cx_cursors[i].kind==CXCursor_ClassDecl || cx_cursors[i].kind==CXCursor_StructDecl) {
+    else if(!tu_cursors && (cx_cursors[i].kind==CXCursor_ClassDecl || cx_cursors[i].kind==CXCursor_StructDecl)) {
       Token token(cx_tu, cx_tokens[i], cx_cursors[i]);
       auto cursor=token.get_cursor();
       auto token_offsets=token.get_source_range().get_offsets();
-      if(token_offsets.second!=cursor.get_source_range().get_offsets().second && token_offsets.first!=cursor.get_source_location().get_offset() && token.is_identifier()) {
+      if(token_offsets.second!=cursor.get_source_range().get_offsets().second && token_offsets.first!=cursor.get_source_location().get_offset() && token.is_identifier())
+        invalid_tokens.emplace(i, token_offsets.first);
+    }
+    
+    emplace_back(Token(cx_tu, cx_tokens[i], cx_cursors[i]));
+  }
+  if(!tu_cursors && !invalid_tokens.empty()) {
     class VisitorData {
     public:
-          std::string path;
+      Tokens *tokens;
-          bool found_path;
+      const std::string &path;
-          std::pair<Offset, Offset> offsets;
+      std::map<size_t, Offset> &invalid_tokens;
-          CXCursor found_cursor;
+      std::vector<Cursor> cursors;
     };
-        VisitorData data{token.get_source_location().get_path(), false, token_offsets, clang_getNullCursor()};
+    VisitorData data{this, range.get_start().get_path(), invalid_tokens, {}};
     auto translation_unit_cursor = clang_getTranslationUnitCursor(cx_tu);
     clang_visitChildren(translation_unit_cursor, [](CXCursor cx_cursor, CXCursor cx_parent, CXClientData data_) {
       auto data=static_cast<VisitorData*>(data_);
       Cursor cursor(cx_cursor);
-          if(data->found_path || cursor.get_source_location().get_path()==data->path) {
+      if(cursor.get_source_location().get_path()==data->path)
-            data->found_path=true;
+        data->cursors.emplace_back(cursor);
-            if(cursor.get_source_range().get_offsets().second==data->offsets.second && cursor.get_source_location().get_offset()==data->offsets.first) {
-              data->found_cursor=cx_cursor;
-              return CXChildVisit_Break;
-            }
-            return CXChildVisit_Recurse;
-          }
       return CXChildVisit_Continue;
     }, &data);
     
-        if(!clang_Cursor_isNull(data.found_cursor))
+    for(auto &cursor: data.cursors) {
-          cx_cursors[i]=data.found_cursor;
+      clang_visitChildren(cursor.cx_cursor, [](CXCursor cx_cursor, CXCursor cx_parent, CXClientData data_) {
+        auto data=static_cast<VisitorData*>(data_);
+        if(clang_getCursorKind(cx_cursor)==CXCursor_FieldDecl) {
+          Cursor cursor(cx_cursor);
+          auto clang_offset=cursor.get_source_location().get_offset();
+          for(auto it=data->invalid_tokens.begin();it!=data->invalid_tokens.end();++it) {
+            if(it->second==clang_offset) {
+              (*data->tokens)[it->first].cx_cursor=cursor.cx_cursor;
+              data->invalid_tokens.erase(it);
+              if(data->invalid_tokens.empty())
+                return CXChildVisit_Break;
+              break;
             }
           }
-    emplace_back(Token(cx_tu, cx_tokens[i], cx_cursors[i]));
+        }
+        return CXChildVisit_Recurse;
+      }, &data);
+    }
   }
 }