From 2ff039022be7441b359e11e16144d033bf3e7580 Mon Sep 17 00:00:00 2001 From: Mirek Fidler Date: Sun, 31 Jul 2022 20:53:48 +0200 Subject: [PATCH] developing indexer --- uppsrc/Core/Topt.h | 48 ++--- uppsrc/ide/Core/Hdepend.cpp | 1 + uppsrc/ide/clang/Indexer.cpp | 123 +++++++++--- uppsrc/ide/clang/Visitor.cpp | 366 ++++++++++++++++++++++------------- uppsrc/ide/clang/clang.cpp | 18 -- uppsrc/ide/clang/clang.h | 35 ++-- uppsrc/ide/clang/todo.txt | 2 + 7 files changed, 371 insertions(+), 222 deletions(-) diff --git a/uppsrc/Core/Topt.h b/uppsrc/Core/Topt.h index 16b0f1a8c..b8c6cf76f 100644 --- a/uppsrc/Core/Topt.h +++ b/uppsrc/Core/Topt.h @@ -429,29 +429,6 @@ hash_t memhash(const void *ptr, size_t size); template inline hash_t GetHashValue(const T& x) { return x.GetHashValue(); } -struct CombineHash { - hash_t hash; - - template CombineHash& Do(const T& x) { Put(GetHashValue(x)); return *this; } - -public: - CombineHash& Put(hash_t h) { hash = HASH_CONST2 * hash + h; return *this; } - - operator hash_t() const { return hash; } - - CombineHash() { hash = HASH_CONST1; } - template - CombineHash(const T& h1) { hash = HASH_CONST1; Do(h1); } - template - CombineHash(const T& h1, const U& h2) { hash = HASH_CONST1; Do(h1); Do(h2); } - template - CombineHash(const T& h1, const U& h2, const V& h3) { hash = HASH_CONST1; Do(h1); Do(h2); Do(h3); } - template - CombineHash(const T& h1, const U& h2, const V& h3, const W& h4) { hash = HASH_CONST1; Do(h1); Do(h2); Do(h3); Do(h4); } - - template CombineHash& operator<<(const T& x) { Do(x); return *this; } -}; - template<> inline hash_t GetHashValue(const char& a) { return (hash_t)a; } template<> inline hash_t GetHashValue(const signed char& a) { return (const hash_t)a; } template<> inline hash_t GetHashValue(const unsigned char& a) { return (const hash_t)a; } @@ -478,12 +455,35 @@ template<> inline hash_t GetHashValue(const double& a) { return memhash( #ifdef CPU_32 inline hash_t GetPtrHashValue(const void *a) { return (int)a; } #else -inline hash_t GetPtrHashValue(const void *a) { return CombineHash((hash_t)(uintptr_t)a); } +inline hash_t GetPtrHashValue(const void *a) { return (hash_t)(uintptr_t)a; } #endif template inline hash_t GetHashValue(T *ptr) { return GetPtrHashValue(reinterpret_cast(ptr)); } +struct CombineHash { + hash_t hash; + + template CombineHash& Do(const T& x) { Put(GetHashValue(x)); return *this; } + +public: + CombineHash& Put(hash_t h) { hash = HASH_CONST2 * hash + h; return *this; } + + operator hash_t() const { return hash; } + + CombineHash() { hash = HASH_CONST1; } + template + CombineHash(const T& h1) { hash = HASH_CONST1; Do(h1); } + template + CombineHash(const T& h1, const U& h2) { hash = HASH_CONST1; Do(h1); Do(h2); } + template + CombineHash(const T& h1, const U& h2, const V& h3) { hash = HASH_CONST1; Do(h1); Do(h2); Do(h3); } + template + CombineHash(const T& h1, const U& h2, const V& h3, const W& h4) { hash = HASH_CONST1; Do(h1); Do(h2); Do(h3); Do(h4); } + + template CombineHash& operator<<(const T& x) { Do(x); return *this; } +}; + template struct Data_S_ : Moveable< Data_S_ > { diff --git a/uppsrc/ide/Core/Hdepend.cpp b/uppsrc/ide/Core/Hdepend.cpp index be4d899a6..b33fe9d79 100644 --- a/uppsrc/ide/Core/Hdepend.cpp +++ b/uppsrc/ide/Core/Hdepend.cpp @@ -335,6 +335,7 @@ Time Hdepend::FileTime(int ii) Vector Hdepend::GetDependencies(const String& path, bool bydefine_too) { + DTIMING("GetDependecies"); ClearFlag(); Index out; out.Add(File(path)); diff --git a/uppsrc/ide/clang/Indexer.cpp b/uppsrc/ide/clang/Indexer.cpp index 5e51040bd..e4202922f 100644 --- a/uppsrc/ide/clang/Indexer.cpp +++ b/uppsrc/ide/clang/Indexer.cpp @@ -6,6 +6,20 @@ #define LDUMP(x) //DDUMP(x) #define LDUMPM(x) //DDUMPM(x) +struct Timest { // TODO remove + String name; + int tm; + Timest(const String& name) : name(name) { tm = msecs(); } + ~Timest() { + tm = msecs() - tm; + if(tm > 500) + DLOG(name << " " << tm / 1000.0); + } +}; + +#define ITIMESTOP(x) Timest COMBINE(sTmStop, __LINE__)(x); + + String FindMasterSource(Hdepend& hdepend, const Workspace& wspc, const String& header_file) { String master_source; @@ -107,7 +121,9 @@ ArrayMap& CodeIndex() void DumpIndex() { + ITIMESTOP("DumpIndex"); GuiLock __; + ITIMESTOP("DumpIndex2"); FileOut out(ConfigFile("current_index.dump")); ArrayMap& x = CodeIndex(); for(const auto& m : ~x) { @@ -118,37 +134,47 @@ void DumpIndex() } CoEvent Indexer::event; -Hdepend Indexer::hdepend; +CoEvent Indexer::scheduler; Mutex Indexer::mutex; Vector Indexer::jobs; int Indexer::jobi; std::atomic Indexer::running_indexers; VectorMap Indexer::master_file; +String Indexer::main; +String Indexer::includes; +String Indexer::defines; void Indexer::IndexerThread() { while(!Thread::IsShutdownThreads()) { Clang clang; int tm0 = msecs(); - running_indexers++; bool was_job = false; // for diagnostics - for(;;) { + ++running_indexers; + while(!Thread::IsShutdownThreads()) { Job job; { - DTIMESTOP("Acquire job"); + ITIMESTOP("Acquire job"); Mutex::Lock __(mutex); - if(jobi >= jobs.GetCount()) + if(jobi < jobs.GetCount()) + job = jobs[jobi++]; + else break; - job = jobs[jobi++]; was_job = true; } + + if(Thread::IsShutdownThreads()) + break; + + // if(job.path != "C:\\upp\\ide$$$blitz.cpp") + // continue; { - DTIMESTOP("Parsing " + job.path + " " + AsString(job.file_times)); + ITIMESTOP("Parsing " + job.path + " " + AsString(job.file_times)); clang.Parse(job.path, job.blitz, job.includes, job.defines, CXTranslationUnit_DetailedPreprocessingRecord| CXTranslationUnit_KeepGoing| - CXTranslationUnit_SkipFunctionBodies| + // CXTranslationUnit_SkipFunctionBodies| (job.blitz.GetCount() ? 0 : PARSE_FILE)); // DumpDiagnostics(clang.tu); } @@ -165,6 +191,7 @@ void Indexer::IndexerThread() VectorMap do_file_cache; v.WhenFile = [&](const String& path) { + DTIMING("WhenFile"); if(IsNull(path)) return false; if(current_file != path) { @@ -178,6 +205,7 @@ void Indexer::IndexerThread() current_file = path; int q = do_file_cache.Find(path); if(q < 0) { + DTIMING("WhenFile 2"); Mutex::Lock __(mutex); do_file = job.file_times.Find(master_file.Get(NormalizePath(path), Null)) >= 0; do_file_cache.Add(path, do_file); @@ -188,10 +216,14 @@ void Indexer::IndexerThread() } return do_file; }; - + + ITIMESTOP("Visitor " + job.path + " " + AsString(job.file_times)); v.Do(clang.tu); } + if(Thread::IsShutdownThreads()) + break; + for(const auto& m : ~job.file_times) // in create entries even if there are no items to avoid recompiling v.item.GetAdd(NormalizePath(m.key)); @@ -202,45 +234,72 @@ void Indexer::IndexerThread() f.includes = job.includes; f.items = pick(m.value); f.time = job.file_times.Get(path, Time::Low()); + ITIMESTOP("Save"); SaveChangedFile(CachedAnnotationPath(path, f.defines, f.includes, master_file.Get(path, Null)), StoreAsString(f), true); + ITIMESTOP("Set"); GuiLock __; CodeIndex().GetAdd(path) = pick(f); } + + } + bool last = false; + { + Mutex::Lock __(mutex); + if(--running_indexers == 0 && jobs.GetCount()) { + DLOG("Done everything " << (msecs() - tm0) / 1000.0 << " s"); + jobs.Clear(); + scheduler.Broadcast(); + last = true; + } } #ifdef _DEBUG - if(--running_indexers == 0 && was_job) { - DLOG("Done everything " << (msecs() - tm0) / 1000.0 << " s"); + if(last) DumpIndex(); // TODO remove - } #endif + if(Thread::IsShutdownThreads()) + break; event.Wait(); LLOG("Indexers Thread::IsShutdownThreads() " << Thread::IsShutdownThreads()); } LLOG("Exiting IndexerThread"); } -void Indexer::Start(const String& main, const String& includes_, const String& defines) +void Indexer::Start(const String& main, const String& includes, const String& defines) { - LLOG("Indexer::Start =============================== "); - ONCELOCK { MemoryIgnoreNonMainLeaks(); MemoryIgnoreNonUppThreadsLeaks(); // clangs leaks static memory in threads Thread::AtShutdown([] { LLOG("Shutdown indexers"); event.Broadcast(); + scheduler.Broadcast(); }); for(int i = 0; i < CPU_Cores(); i++) // TODO: CPU_Cores? Thread::StartNice([] { Indexer::IndexerThread(); }); + Thread::StartNice([] { SchedulerThread(); }); } - Thread::Start([=] { - String includes = Merge(";", includes_, GetClangInternalIncludes()); + GuiLock __; + Indexer::main = main; + Indexer::includes = includes; + Indexer::defines = defines; + if(jobs.GetCount() == 0) + scheduler.Broadcast(); +} + +void Indexer::SchedulerThread() +{ + Hdepend hdepend; + while(!Thread::IsShutdownThreads()) { + scheduler.Wait(); + + Mutex::Lock __(mutex); + String includes, defines; VectorMap>> sources; { GuiLock __; - DTIMESTOP("Load workspace"); + ITIMESTOP("Load workspace"); Workspace wspc; wspc.Scan(main); @@ -254,12 +313,14 @@ void Indexer::Start(const String& main, const String& includes_, const String& d ps.Add({ NormalizePath(path), pk[i].noblitz }); } } + includes = Merge(";", Indexer::includes, GetClangInternalIncludes()); + defines = Indexer::defines; } { - DTIMESTOP("Create indexer jobs"); + ITIMESTOP("Create indexer jobs"); Mutex::Lock __(mutex); - DTIMESTOP("Create indexer jobs after lock"); + ITIMESTOP("Create indexer jobs after lock"); hdepend.NoConsole(); LDUMP(includes); hdepend.SetDirs(includes); @@ -267,9 +328,10 @@ void Indexer::Start(const String& main, const String& includes_, const String& d jobs.Clear(); { - DTIMESTOP("Master files"); // <<<< this is slow + ITIMESTOP("Master files"); // <<<< this is slow master_file.Clear(); - for(const auto& pk : sources) + for(int i = sources.GetCount() - 1; i >= 0; i--) { + const auto& pk = sources[i]; for(const auto& f : pk) { master_file.Add(f.a, f.a); for(String p : hdepend.GetDependencies(f.a)) { @@ -278,12 +340,13 @@ void Indexer::Start(const String& main, const String& includes_, const String& d master_file.Add(p, f.a); } } + } } - LDUMPM(master_file); + DUMPM(master_file); { - DTIMESTOP("Loading from cache"); + ITIMESTOP("Loading from cache"); for(String path : master_file.GetKeys()) { FileAnnotation0 f; { @@ -305,7 +368,7 @@ void Indexer::Start(const String& main, const String& includes_, const String& d } { // remove files that are not in project anymore - DTIMESTOP("Removing files"); + ITIMESTOP("Removing files"); GuiLock __; for(int i = 0; i < CodeIndex().GetCount(); i++) if(master_file.Find(CodeIndex().GetKey(i)) < 0) @@ -313,7 +376,7 @@ void Indexer::Start(const String& main, const String& includes_, const String& d CodeIndex().Sweep(); } - DTIMESTOP("Create indexer jobs2"); + ITIMESTOP("Create indexer jobs2"); jobi = 0; for(const auto& pkg : ~sources) { String blitz; @@ -349,7 +412,9 @@ void Indexer::Start(const String& main, const String& includes_, const String& d } } } - - event.Broadcast(); - }); + if(jobs.GetCount()) { + DLOG("======= Unleash indexers"); + event.Broadcast(); + } + } } diff --git a/uppsrc/ide/clang/Visitor.cpp b/uppsrc/ide/clang/Visitor.cpp index 459a9b763..1f9208d9b 100644 --- a/uppsrc/ide/clang/Visitor.cpp +++ b/uppsrc/ide/clang/Visitor.cpp @@ -2,11 +2,194 @@ // #define DUMPTREE +class ClangCursorInfo { + CXCursor cursor; + CXCursorKind cursorKind; + CXCursor parent; + CXCursorKind parentKind; + CXPrintingPolicy pp_id; + + + bool hasraw_id = false; + String raw_id; + + bool hasscope = false; + String scope; + String nspace; + + bool hastype = false; + String type; + + bool hasname = false; + String name; + + bool hasid = false; + String id; + +public: + int Kind() { return cursorKind; } + String RawId(); + String Scope(); + String Nspace() { Scope(); return nspace; } + String Type(); + String Name(); + bool NoId(); + String Id(); + + ClangCursorInfo(CXCursor cursor, CXPrintingPolicy pp_id); +}; + +ClangCursorInfo::ClangCursorInfo(CXCursor cursor, CXPrintingPolicy pp_id) +: cursor(cursor), pp_id(pp_id) { + cursorKind = clang_getCursorKind(cursor); + parent = clang_getCursorSemanticParent(cursor); + parentKind = clang_getCursorKind(parent); +} + +force_inline +String ClangCursorInfo::RawId() +{ + if(!hasraw_id) { + raw_id = FetchString(clang_getCursorPrettyPrinted(cursor, pp_id)); + hasraw_id = true; + } + return raw_id; +} + +force_inline +String ClangCursorInfo::Type() +{ + if(!hastype) { + type = GetTypeSpelling(cursor); + hastype = true; + } + return type; +} + +force_inline +String ClangCursorInfo::Name() +{ + if(!hasname) { + name = GetCursorSpelling(cursor); + hasname = true; + } + return name; +} + +String ClangCursorInfo::Scope() +{ + if(!hasscope) { + CXCursor p = parent; + for(;;) { + CXCursorKind k = clang_getCursorKind(p); + if(k != CXCursor_EnumDecl) { + if(findarg(k, CXCursor_Namespace, CXCursor_ClassTemplate, CXCursor_StructDecl, CXCursor_UnionDecl, CXCursor_ClassDecl) < 0) + break; + String q = GetCursorSpelling(p); + scope = scope.GetCount() ? q + "::" + scope : q; + if(k == CXCursor_Namespace) + nspace = nspace.GetCount() ? q + "::" + nspace : q; + } + p = clang_getCursorSemanticParent(p); + } + int q = scope.Find('('); // 'Struct::(unnamed enum at C:\u\upp.src\upptst\Annotations\main.cpp:47:2)' + if(q >= 0) + scope.Trim(q); + if(scope.GetCount() && *scope.Last() != ':') + scope << "::"; + hasscope = true; + } + return scope; +} + +bool ClangCursorInfo::NoId() +{ // TODO: remove + return findarg(parentKind, CXCursor_FunctionTemplate, CXCursor_FunctionDecl, CXCursor_CXXMethod, + CXCursor_Constructor, CXCursor_Destructor) >= 0 || + findarg(cursorKind, CXCursor_StructDecl, CXCursor_UnionDecl, CXCursor_ClassDecl, CXCursor_FunctionTemplate, + CXCursor_FunctionDecl, CXCursor_Constructor, CXCursor_Destructor, CXCursor_CXXMethod, + CXCursor_VarDecl, CXCursor_FieldDecl, CXCursor_ClassTemplate, + CXCursor_ConversionFunction, CXCursor_MacroDefinition, + CXCursor_EnumConstantDecl) < 0; +} + +String ClangCursorInfo::Id() +{ + if(!hasid) { + String m; + if(findarg(parentKind, CXCursor_FunctionTemplate, CXCursor_FunctionDecl, CXCursor_CXXMethod, + CXCursor_Constructor, CXCursor_Destructor) < 0) { // local variable, TODO (members of local structure) + switch(cursorKind) { + case CXCursor_StructDecl: + case CXCursor_UnionDecl: + case CXCursor_ClassDecl: + m = Type(); + break; + case CXCursor_FunctionTemplate: + case CXCursor_FunctionDecl: + case CXCursor_Constructor: + case CXCursor_Destructor: + case CXCursor_CXXMethod: + m = RawId(); + break; + case CXCursor_VarDecl: + case CXCursor_FieldDecl: + case CXCursor_ClassTemplate: + m << Scope() << Name(); + break; + case CXCursor_ConversionFunction: + m << Scope() << "operator " << Type(); + break; + case CXCursor_MacroDefinition: + m = Name(); + break; + case CXCursor_EnumConstantDecl: + m << Scope() << Name(); + break; + /* + case CXCursor_EnumDecl: + case CXCursor_ParmDecl: + case CXCursor_TypedefDecl: + case CXCursor_Namespace: + case CXCursor_UnexposedDecl: + case CXCursor_NamespaceAlias: + break; + */ + default:; + } + } + id = CleanupId(m); + hasid = true; + } + return id; +} + +SourceLocation ClangVisitor::GetLocation(CXSourceLocation cxlocation) +{ + CXFile file; + SourceLocation p; + unsigned line; + unsigned column; + unsigned offset; + clang_getExpansionLocation(cxlocation, &file, &line, &column, &offset); + p.pos.x = int(column - 1); + p.pos.y = int(line - 1); + + int q = cxfile.Find(file); + if(q >= 0) + p.path = cxfile[q]; + else { + p.path = NormalizePath(FetchString(clang_getFileName(file))); + cxfile.Add(file, p.path); + } + return p; +} + bool ClangVisitor::ProcessNode(CXCursor cursor) { CXSourceLocation cxlocation = clang_getCursorLocation(cursor); - - CXCursorKind cursorKind = clang_getCursorKind(cursor); + + ClangCursorInfo ci(cursor, pp_id); #ifdef DUMPTREE _DBG_ @@ -34,134 +217,47 @@ bool ClangVisitor::ProcessNode(CXCursor cursor) } #endif - String pid = FetchString(clang_getCursorPrettyPrinted(cursor, pp_id)); - String id; - String name; - String nspace; - - auto MakeCursorInfo = [&](CXCursor cursor) { - String m; - - CXCursor parent = clang_getCursorSemanticParent(cursor); - CXCursorKind parentKind = clang_getCursorKind(parent); - - name = GetCursorSpelling(cursor); - String type = GetTypeSpelling(cursor); - - CXCursor p = parent; - String scope; - nspace.Clear(); - for(;;) { - CXCursorKind k = clang_getCursorKind(p); - if(k != CXCursor_EnumDecl) { - if(findarg(k, CXCursor_Namespace, CXCursor_ClassTemplate, CXCursor_StructDecl, CXCursor_UnionDecl, CXCursor_ClassDecl) < 0) - break; - String q = GetCursorSpelling(p); - scope = scope.GetCount() ? q + "::" + scope : q; - if(k == CXCursor_Namespace) - nspace = nspace.GetCount() ? q + "::" + nspace : q; - } - p = clang_getCursorSemanticParent(p); - } - int q = scope.Find('('); // 'Struct::(unnamed enum at C:\u\upp.src\upptst\Annotations\main.cpp:47:2)' - if(q >= 0) - scope.Trim(q); - if(scope.GetCount() && *scope.Last() != ':') - scope << "::"; - - if(findarg(parentKind, CXCursor_FunctionTemplate, CXCursor_FunctionDecl, CXCursor_CXXMethod, - CXCursor_Constructor, CXCursor_Destructor) < 0) { // local variable, TODO (members of local structure) - switch(cursorKind) { - case CXCursor_StructDecl: - case CXCursor_UnionDecl: - case CXCursor_ClassDecl: - m = type; - break; - case CXCursor_FunctionTemplate: - case CXCursor_FunctionDecl: - case CXCursor_Constructor: - case CXCursor_Destructor: - case CXCursor_CXXMethod: - m = pid; - break; - case CXCursor_VarDecl: - case CXCursor_FieldDecl: - case CXCursor_ClassTemplate: - m << scope << name; - break; - case CXCursor_ConversionFunction: - m << scope << "operator " << type; - break; - case CXCursor_MacroDefinition: - m = name; - break; - case CXCursor_EnumConstantDecl: - m << scope << name; - break; - /* - case CXCursor_EnumDecl: - case CXCursor_ParmDecl: - case CXCursor_TypedefDecl: - case CXCursor_Namespace: - case CXCursor_UnexposedDecl: - case CXCursor_NamespaceAlias: - break; - */ - default:; - } - } - id = CleanupId(m); + + SourceLocation loc; + bool loc_loaded = false; + auto LoadLocation = [&] { + if(loc_loaded) return; + loc = GetLocation(cxlocation); + loc_loaded = true; }; - - bool position_loaded = false; - unsigned line; - unsigned column; - unsigned offset; - String path; - - auto LoadPosition = [&] { - if(position_loaded) return; - CXFile file; - clang_getExpansionLocation(cxlocation, &file, &line, &column, &offset); - path = FetchString(clang_getFileName(file)); - position_loaded = true; - }; - - if(WhenFile) - LoadPosition(); - - if(findarg(cursorKind, CXCursor_CXXMethod, CXCursor_FunctionTemplate) >= 0) { - LoadPosition(); - tfn.GetAdd(MakeTuple(path, line)) = pid; + + if(findarg(ci.Kind(), CXCursor_CXXMethod, CXCursor_FunctionTemplate) >= 0) { + LoadLocation(); + tfn.GetAdd(loc).cursor = cursor; } - if(!(WhenFile ? WhenFile(path) : clang_Location_isFromMainFile(cxlocation))) - return findarg(cursorKind, CXCursor_StructDecl, CXCursor_UnionDecl, CXCursor_ClassDecl, + if(WhenFile) + LoadLocation(); + + if(!(WhenFile ? WhenFile(loc.path) : clang_Location_isFromMainFile(cxlocation))) + return findarg(ci.Kind(), CXCursor_StructDecl, CXCursor_UnionDecl, CXCursor_ClassDecl, CXCursor_FunctionTemplate, CXCursor_FunctionDecl, CXCursor_Constructor, CXCursor_Destructor, CXCursor_ClassTemplatePartialSpecialization, CXCursor_UnexposedDecl, CXCursor_UsingDeclaration, CXCursor_VarDecl, CXCursor_EnumConstantDecl, CXCursor_TypeAliasTemplateDecl, CXCursor_EnumDecl, CXCursor_ConversionFunction) < 0; - MakeCursorInfo(cursor); - CXCursor ref = clang_getCursorReferenced(cursor); - bool reference = !clang_Cursor_isNull(ref); - if(id.GetCount() || reference) - LoadPosition(); - - Point pos(column - 1, line - 1); + String id = ci.Id(); + DHITCOUNT("Resolved ID"); if(id.GetCount()) { - AnnotationItem& r = item.GetAdd(path).Add(); - r.kind = cursorKind; - r.name = name; - r.line = line - 1; + DTIMING("Has ID"); + LoadLocation(); + AnnotationItem& r = item.GetAdd(loc.path).Add(); + r.kind = ci.Kind(); + r.name = ci.Name(); + r.line = loc.pos.y; r.id = id; - r.pretty = cursorKind == CXCursor_MacroDefinition ? r.name + r.pretty = ci.Kind() == CXCursor_MacroDefinition ? r.name : CleanupPretty(FetchString(clang_getCursorPrettyPrinted(cursor, pp_pretty))); r.definition = clang_isCursorDefinition(cursor); - r.nspace = nspace; + r.nspace = ci.Nspace(); if(findarg(r.kind, CXCursor_Constructor, CXCursor_Destructor) >= 0) { int q = r.id.Find('('); if(q >= 0) { @@ -176,36 +272,32 @@ bool ClangVisitor::ProcessNode(CXCursor cursor) r.nest = r.id.Mid(0, q); r.nest.TrimEnd("::"); } - if(IsStruct(cursorKind)) - MergeWith(r.nest, "::", name); + if(IsStruct(ci.Kind())) + MergeWith(r.nest, "::", r.name); } - r.uname = ToUpper(name); + r.uname = ToUpper(r.name); r.unest = ToUpper(r.nest); - ReferenceItem rm; - rm.pos = pos; + ReferenceItem rm; // prevent self-references + rm.pos = loc.pos; rm.id = r.id; - ref_done.GetAdd(path).FindAdd(rm); + ref_done.GetAdd(loc.path).FindAdd(rm); } - if(reference) { - cursorKind = clang_getCursorKind(ref); - pid = FetchString(clang_getCursorPrettyPrinted(ref, pp_id)); - if(pid.Find('<') >= 0) { // might be a template specialization - CXFile file; - unsigned line; - unsigned column; - unsigned offset; - clang_getExpansionLocation(clang_getCursorLocation(ref), &file, &line, &column, &offset); - pid = tfn.Get(MakeTuple(FetchString(clang_getFileName(file)), line), pid); - } - MakeCursorInfo(ref); + if(!clang_Cursor_isNull(ref)) { + DTIMING("Ref"); + LoadLocation(); + SourceLocation ref_loc = GetLocation(clang_getCursorLocation(ref)); + int q = tfn.Find(ref_loc); + + ClangCursorInfo ref_ci(q >= 0 ? tfn[q].cursor : ref, pp_id); + ReferenceItem rm; - rm.pos = pos; - rm.id = id; - Index& rd = ref_done.GetAdd(path); + rm.pos = ref_loc.pos; + rm.id = ref_ci.Id(); + Index& rd = ref_done.GetAdd(ref_loc.path); if(rm.id.GetCount() && rd.Find(rm) < 0) { rd.Add(rm); - refs.GetAdd(path).Add(rm); + refs.GetAdd(ref_loc.path).Add(rm); } } return true; diff --git a/uppsrc/ide/clang/clang.cpp b/uppsrc/ide/clang/clang.cpp index 1d56f353a..c5d1a6c29 100644 --- a/uppsrc/ide/clang/clang.cpp +++ b/uppsrc/ide/clang/clang.cpp @@ -25,11 +25,6 @@ String GetTypeSpelling(CXCursor cursor) return FetchString(clang_getTypeSpelling(clang_getCursorType(cursor))); } -String SourceLocation::ToString() const -{ - return String() << filename << " (" << line << ":" << column << ")"; -} - String GetClangInternalIncludes() { static String includes; @@ -56,19 +51,6 @@ String GetClangInternalIncludes() return includes; } -SourceLocation::SourceLocation(CXSourceLocation location) -{ - CXFile file; - unsigned line_; - unsigned column_; - unsigned offset_; - clang_getExpansionLocation(location, &file, &line_, &column_, &offset_); - line = line_; - column = column_; - offset = offset_; - filename = FetchString(clang_getFileName(file)); -} - void Clang::Dispose() { if(tu) clang_disposeTranslationUnit(tu); diff --git a/uppsrc/ide/clang/clang.h b/uppsrc/ide/clang/clang.h index a5cd9eba8..815c35015 100644 --- a/uppsrc/ide/clang/clang.h +++ b/uppsrc/ide/clang/clang.h @@ -21,15 +21,14 @@ String GetCursorKindName(CXCursorKind cursorKind); String GetCursorSpelling(CXCursor cursor); String GetTypeSpelling(CXCursor cursor); -struct SourceLocation { - int line; - int column; - int offset; - String filename; +struct SourceLocation : Moveable { + String path; + Point pos; - String ToString() const; - - SourceLocation(CXSourceLocation location); + bool operator==(const SourceLocation& b) const { return path == b.path && pos == b.pos; } + bool operator!=(const SourceLocation& b) const { return !operator==(b); } + void Serialize(Stream& s) { s % path % pos; } + hash_t GetHashValue() const { return CombineHash(path, pos); } }; String RedefineMacros(); @@ -88,7 +87,7 @@ struct AutoCompleteItem : Moveable { struct AnnotationItem : Moveable { int kind; - int line; + int line; // TODO: Point bool definition; String name; // Method String id; // Upp::Class::Method(Upp::Point p) @@ -151,12 +150,19 @@ class ClangVisitor { bool initialized = false; CXPrintingPolicy pp_id, pp_pretty; - bool ProcessNode(CXCursor c); + bool ProcessNode(CXCursor c); + SourceLocation GetLocation(CXSourceLocation c); friend CXChildVisitResult clang_visitor(CXCursor cursor, CXCursor p, CXClientData clientData); - + + VectorMap cxfile; // accelerate CXFile (CXFile has to be valid across tree as there is no Dispose) VectorMap> ref_done; // avoid self-references, multiple references - VectorMap, String> tfn; // to convert e.g. Index::Find(String) to Index::Find(T) + + struct MCXCursor : Moveable { + CXCursor cursor; + }; + + ArrayMap tfn; // to convert e.g. Index::Find(String) to Index::Find(T) public: VectorMap> item; @@ -202,15 +208,16 @@ class Indexer { WithDeepCopy> file_times; }; - static CoEvent event; - static Hdepend hdepend; + static CoEvent event, scheduler; static Mutex mutex; static Vector jobs; static int jobi; static std::atomic running_indexers; static VectorMap master_file; // header -> first file that includes it + static String main, includes, defines; static void IndexerThread(); + static void SchedulerThread(); public: static void Start(const String& main, const String& includes, const String& defines); diff --git a/uppsrc/ide/clang/todo.txt b/uppsrc/ide/clang/todo.txt index eef15876b..c19d5ff85 100644 --- a/uppsrc/ide/clang/todo.txt +++ b/uppsrc/ide/clang/todo.txt @@ -124,6 +124,8 @@ GREAT CODEBASE PURGE: - Implement annotation columns +- Indexer::IsRunning should account for SchedulerThread + NONCLANG: