From 69f1d52ea6f411ea0fc9089f5e7462d886d43a26 Mon Sep 17 00:00:00 2001
From: Mirek Fidler <mirek.fidler@gmail.com>
Date: Wed, 3 Aug 2022 20:26:46 +0200
Subject: [PATCH] hdepend2

---
 uppsrc/ide/Assist.cpp        |  21 ++-
 uppsrc/ide/Assist.h          |   1 +
 uppsrc/ide/Core/Core.h       |  47 +++++
 uppsrc/ide/Core/Hdepend2.cpp | 321 ++++++++++++++++++++++++++++-------
 uppsrc/ide/IncludeTrick.cpp  |  16 +-
 uppsrc/ide/clang/Indexer.cpp |  43 +++++
 uppsrc/ide/clang/clang.h     |   1 +
 uppsrc/ide/clang/todo.txt    |  12 +-
 8 files changed, 400 insertions(+), 62 deletions(-)

diff --git a/uppsrc/ide/Assist.cpp b/uppsrc/ide/Assist.cpp
index 1898c9b17..87378c500 100644
--- a/uppsrc/ide/Assist.cpp
+++ b/uppsrc/ide/Assist.cpp
@@ -805,8 +805,25 @@ bool AssistEditor::Key(dword key, int count)
 		Exclamation("No annotation for this line.");
 	}
 	if(key == K_F11) {
-		Indexer::Start(theide->main, theide->GetCurrentIncludePath(), theide->GetCurrentDefines());
-	}
+/*		Workspace wspc;
+		wspc.Scan(theide->main);
+		
+		DDUMP(Merge(";", theide->GetCurrentIncludePath(), GetClangInternalIncludes()));
+
+		PPInfo ppi;
+		ppi.SetIncludes(Merge(";", theide->GetCurrentIncludePath(), GetClangInternalIncludes()));
+		Index<String> files;
+		for(int pi = 0; pi < wspc.GetCount(); pi++) {
+			String pk_name = wspc[pi];
+			const Package& pk = wspc.GetPackage(pi);
+			for(int i = 0; i < pk.GetCount(); i++) {
+				String path = SourcePath(pk_name, pk[i]);
+				ppi.GatherDependencies(path, files);
+			}
+		}
+		
+		DDUMP(files);
+*/	}
 #endif
 	if(popup.IsOpen()) {
 		int k = key & ~K_CTRL;
diff --git a/uppsrc/ide/Assist.h b/uppsrc/ide/Assist.h
index 3bc20fabb..bc76c6633 100644
--- a/uppsrc/ide/Assist.h
+++ b/uppsrc/ide/Assist.h
@@ -150,6 +150,7 @@ struct AssistEditor : CodeEditor, Navigator {
 	bool      navigator_right = true;
 	
 	Hdepend   hdepend;
+	PPInfo    hdepend2;
 	String    master_source;
 
 	CurrentFileContext CurrentContext();
diff --git a/uppsrc/ide/Core/Core.h b/uppsrc/ide/Core/Core.h
index fee4bad08..1ef7c9b6d 100644
--- a/uppsrc/ide/Core/Core.h
+++ b/uppsrc/ide/Core/Core.h
@@ -97,6 +97,53 @@ public:
 	void                  Serialize(Stream& s);
 };
 
+class PPInfo {
+	enum { AUTO, APPROVED, PROHIBITED };
+	struct PPFile : Moveable<PPFile> {
+		VectorMap<String, int>    flags; // "#if... flagXXXX"
+		VectorMap<String, String> defines; // #define ...
+		Index<String>             includes;
+		Index<String>             define_includes; // #define LAYOUTFILE
+		bool                      guarded; // has include guards
+		int                       blitz; // AUTO, APPROVED, PROHIBITED
+		Time                      time = Null; // file time
+		
+		bool                      dirty = true; // need to be rechecked
+		
+		void Dirty()                          { dirty = true; time = Null; }
+		void Parse(Stream& in);
+		void Serialize(Stream& s)             { s % time % flags % defines % includes % define_includes % guarded % blitz; }
+	};
+	
+	ArrayMap<String, PPFile>                   files;
+	Vector<String>                             includes; // include dirs
+	VectorMap<String, String>                  inc_cache; // cache for FindIncludeFile
+	VectorMap<String, VectorMap<String, Time>> dir_cache; // cache for GetFileTime, FileExists
+
+	PPFile& File(const String& path);
+
+	bool FileExists2(const String& s);
+
+	Time                  GatherDependencies(const String& path, VectorMap<String, Time>& result, Index<String>& define_includes);
+
+public:
+	Event<const String&, const String&> WhenBlitzBlock;
+	Time                  GetFileTime(const String& path);
+	bool                  FileExists(const String& path)                        { return !IsNull(GetFileTime(path)); }
+
+	void                  SetIncludes(const String& includes);
+
+	String                FindIncludeFile(const char *s, const String& filedir, const Vector<String>& incdirs);
+	String                FindIncludeFile(const char *s, const String& filedir);
+
+	bool                  BlitzApproved(const String& path);
+
+	void                  GatherDependencies(const String& path, VectorMap<String, Time>& result);
+	Time                  GetTime(const String& path);
+
+	void                  Dirty();
+};
+
 class IdeContext
 {
 public:
diff --git a/uppsrc/ide/Core/Hdepend2.cpp b/uppsrc/ide/Core/Hdepend2.cpp
index c165de6e5..597bdcfc1 100644
--- a/uppsrc/ide/Core/Hdepend2.cpp
+++ b/uppsrc/ide/Core/Hdepend2.cpp
@@ -1,16 +1,28 @@
 #include "Core.h"
 
-#if 0
+#define LTIMING(x) // RTIMING(x)
 
-struct PPInfo {
-	VectorMap<String, int>    flags; // "#if... flagXXXX"
-	VectorMap<String, String> defines; // #define ...
-	Index<String>             includes;
-	bool                      guarded;
-	
-	
-	void Do(Stream& in);
-};
+using namespace Upp;
+
+void SetSpaces(String& l, int pos, int count)
+{
+	StringBuffer s(l);
+	memset(~s + pos, ' ', count);
+	l = s;
+}
+
+const char *SkipString(const char *s)
+{
+	CParser p(s);
+	try {
+		p.ReadOneString(*s);
+	}
+	catch(CParser::Error) {}
+	s = p.GetPtr();
+	while((byte)*(s - 1) <= ' ')
+		s--;
+	return s;
+}
 
 void RemoveComments(String& l, bool& incomment)
 {
@@ -54,7 +66,7 @@ void RemoveComments(String& l, bool& incomment)
 	}
 }
 
-void PPInfo::Do(Stream& in)
+void PPInfo::PPFile::Parse(Stream& in)
 {
 	LTIMING("PPInfo::Parse");
 
@@ -62,18 +74,33 @@ void PPInfo::Do(Stream& in)
 	defines.Clear();
 	includes.Clear();
 	guarded = false;
+	blitz = AUTO;
 
 	int  linei = 0;
 	bool incomment = false;
 	
 	String guard_id;
 	bool first = true;
-	bool second = false;
 	
 	auto Flag = [&](const String& id) {
 		if(id.StartsWith("flag"))
 			flags.FindAdd(id);
 	};
+
+	auto Blitz = [&](const char *s) {
+		try {
+			CParser p(s);
+			if(p.Id("BLITZ_APPROVE"))
+				blitz = APPROVED;
+			else
+			if(p.Id("BLITZ_PROHIBIT"))
+				blitz = PROHIBITED;
+			else
+			if(p.Id("once"))
+				guarded = true;
+		}
+		catch(CParser::Error) {}
+	};
 	
 	while(!in.IsEof()) {
 		String l = in.GetLine();
@@ -82,69 +109,249 @@ void PPInfo::Do(Stream& in)
 			linei++;
 			l.Cat(in.GetLine());
 		}
+	
+		if(!incomment && l[0] == '/' && l[1] == '/' && l[2] == '#')
+			Blitz(~l + 3);
 		RemoveComments(l, incomment);
 		try {
 			CParser p(l);
 			if(p.Char('#')) {
-				if(do_pp) {
-					if(p.Id("define") && p.IsId()) {
-						p.NoSkipSpaces().NoSkipComments(); // '#define TEST(x)' is different form '#define TEST (x)' - later is parameterless
-						String id = p.ReadId();
-						if(id == guard_id) // TODO: needs to be better
-							guarded = true;
-						if(p.Char('(')) {
-							id << "(";
-							p.SkipSpaces();
-							p.Spaces();
-							bool was = false;
-							while(p.IsId()) {
-								if(was)
-									id << ", ";
-								id << p.ReadId();
-								was = true;
-							}
-							if(p.Char3('.', '.', '.'))
-								id << "...";
-							p.Char(')');
-							id << ")";
-						}
+				if(p.Id("define") && p.IsId()) {
+					p.NoSkipSpaces().NoSkipComments(); // '#define TEST(x)' is different form '#define TEST (x)' - later is parameterless
+					String id = p.ReadId();
+					if(id == guard_id) {
+						DLOG("Guard #define " << id);
+						guarded = true;
+					}
+					if(p.Char('(')) {
+						id << "(";
+						p.SkipSpaces();
 						p.Spaces();
-						defines.Add(id, p.GetPtr());
-					}
-					else
-					if(p.Id("ifndef") && p.IsId()) {
-						String id = p.ReadId();
-						Flag(id);
-						if(first) {
-							guard_id = id;
-							second = true;
+						bool was = false;
+						while(p.IsId()) {
+							if(was)
+								id << ", ";
+							id << p.ReadId();
+							was = true;
 						}
+						if(p.Char3('.', '.', '.'))
+							id << "...";
+						p.Char(')');
+						id << ")";
 					}
-					else
-					if(p.Id("ifdef") && p.IsId()) {
-						String id = p.ReadId();
-						if(id.StartsWith("flag"))
-							flags.Add(id, linei);
+					p.Spaces();
+					defines.Add(id, p.GetPtr());
+				}
+				else
+				if(p.Id("ifndef") && p.IsId()) {
+					String id = p.ReadId();
+					Flag(id);
+					if(first) {
+						DLOG("Guard #ifndef " << id);
+						guard_id = id;
 					}
+				}
+				else
+				if(p.Id("ifdef") && p.IsId()) {
+					Flag(p.ReadId());
+				}
+				else
+				if(p.Id("if"))
+					while(!p.IsEof()) {
+						if(p.IsId())
+							Flag(p.ReadId());
+						else
+							p.Skip();
+					}
+				else
+				if(p.Id("include")) {
+					if(p.IsId())
+						define_includes.Add(p.ReadId());
 					else
-					if(p.Id("if"))
-						while(!p.IsEof()) {
-							if(p.IsId())
-						}
-					if(p.Id("include"))
 						includes.FindAdd(TrimBoth(p.GetPtr()));
 				}
+				else
+				if(p.Id("pragma"))
+					Blitz(p.GetPtr());
 			}
 		}
 		catch(...) {}
-		if(first) {
+		if(first)
 			for(char s : l)
-				if(*s != ' ' || *s != '\t')
+				if(s != ' ' && s != '\t') {
 					first = false;
-		}
+					break;
+				}
 		
 		linei++;
 	}
 }
 
-#endif
\ No newline at end of file
+void PPInfo::SetIncludes(const String& incs)
+{
+	inc_cache.Clear();
+	includes = Split(incs, ';');
+}
+
+Time PPInfo::GetFileTime(const String& path)
+{
+	DTIMING("FileExists");
+	String dir = GetFileFolder(path);
+	String name = GetFileName(path);
+	int q = dir_cache.Find(dir);
+	if(q < 0) {
+		q = dir_cache.GetCount();
+		VectorMap<String, Time>& files = dir_cache.Add(dir);
+		RTIMING("LoadDirCache");
+		for(FindFile ff(dir + "/*.*"); ff; ff.Next())
+			if(ff.IsFile())
+				files.Add(ff.GetName(), ff.GetLastWriteTime());
+	}
+	return dir_cache[q].Get(name, Null);
+}
+
+String PPInfo::FindIncludeFile(const char *s, const String& filedir, const Vector<String>& incdirs)
+{
+	DTIMING("FindIncludeFile");
+	while(*s == ' ' || *s == '\t')
+		s++;
+	int type = *s;
+	if(type == '<' || type == '\"' || type == '?') {
+		s++;
+		String name;
+		if(type == '<') type = '>';
+		while(*s != '\r' && *s != '\n') {
+			if(*s == type) {
+				if(type == '\"') {
+					String fn = NormalizePath(name, filedir);
+					if(FileExists(fn))
+						return fn;
+				}
+				for(int i = 0; i < incdirs.GetCount(); i++) {
+					String fn = NormalizePath(CatAnyPath(incdirs[i], name));
+					if(FileExists(fn))
+						return fn;
+				}
+				break;
+			}
+			name.Cat(*s++);
+		}
+	}
+	return String();
+}
+
+String PPInfo::FindIncludeFile(const char *s, const String& filedir)
+{
+	String key = filedir + "|" + s;
+	int q = inc_cache.Find(key);
+	if(q >= 0)
+		return inc_cache[q];
+	String r = FindIncludeFile(s, filedir, includes);
+	inc_cache.Add(key, r);
+	return r;
+}
+
+void PPInfo::Dirty()
+{
+	for(PPFile& f : files)
+		f.dirty = true;
+	inc_cache.Clear();
+	dir_cache.Clear();
+}
+
+PPInfo::PPFile& PPInfo::File(const String& path)
+{
+	DLOG("PPInfo::File " << path);
+	PPFile& f = files.GetAdd(path);
+	if(f.dirty) {
+		Time tm;
+		
+		tm = GetFileTime(path);
+		if(tm != f.time) {
+			String cache_path = CacheFile(GetFileTitle(path) + "$" + SHA1String(path) + ".ppi");
+
+/* TODO: Just temporary for debugging
+			if(IsNull(f.time)) {
+				DTIMING("Load PPInfo");
+				LoadFromFile(f, cache_path);
+			}
+*/			
+			if(tm != f.time) {
+				FileIn in(path);
+				DLOG("Parse " << path);
+				f.Parse(in); // TODO: If open fails, try to reopen!
+				f.time = tm;
+				StoreToFile(f, cache_path);
+			}
+		}
+		f.dirty = false;
+	}
+	return f;
+}
+
+Time PPInfo::GatherDependencies(const String& path, VectorMap<String, Time>& result, Index<String>& define_includes)
+{
+	PPFile& f = File(path);
+	String dir = GetFileFolder(path);
+	for(const String& i : f.define_includes)
+		define_includes.FindAdd(i);
+	
+	Time ftm = GetFileTime(path);
+
+	auto DoInclude = [&](const String& inc) {
+		String ipath = FindIncludeFile(inc, dir);
+		if(ipath.GetCount() && result.Find(ipath) < 0) {
+			int q = result.GetCount();
+			result.Add(ipath); // prevent infinite recursion
+			result[q] = GetFileTime(ipath); // temporary
+			result[q] = GatherDependencies(ipath, result, define_includes);
+			ftm = max(result[q], ftm);
+		}
+	};
+
+	for(const String& inc : f.includes)
+		DoInclude(inc);
+	for(const String& id : define_includes)
+		for(int q = f.defines.Find(id); q >= 0; q = f.defines.FindNext(q))
+			DoInclude(f.defines[q]);
+	
+	return ftm;
+}
+
+void PPInfo::GatherDependencies(const String& path, VectorMap<String, Time>& result)
+{
+	Index<String> define_includes;
+	GatherDependencies(path, result, define_includes);
+}
+
+Time PPInfo::GetTime(const String& path)
+{
+	VectorMap<String, Time> result;
+	Index<String> define_includes;
+	return GatherDependencies(path, result, define_includes);
+}
+
+bool PPInfo::BlitzApproved(const String& path)
+{
+	PPFile& f = File(path);
+	if(f.blitz == APPROVED)
+		return true;
+	if(f.blitz == PROHIBITED)
+		return false;
+	String dir = GetFileFolder(path);
+	for(const String& inc : f.includes) {
+		String ipath = FindIncludeFile(inc, dir);
+		if(ipath.GetCount()) {
+			PPFile& f = File(ipath);
+			if(f.blitz == APPROVED)
+				return true;
+			if(f.blitz == PROHIBITED)
+				return false;
+			if(!f.guarded) {
+				WhenBlitzBlock(ipath, path);
+				return false;
+			}
+		}
+	}
+	return true;
+}
diff --git a/uppsrc/ide/IncludeTrick.cpp b/uppsrc/ide/IncludeTrick.cpp
index bcd4cbff5..5c086c3b9 100644
--- a/uppsrc/ide/IncludeTrick.cpp
+++ b/uppsrc/ide/IncludeTrick.cpp
@@ -10,8 +10,20 @@ void AssistEditor::SyncHeaders()
 	hdepend.SetDirs(theide->GetCurrentIncludePath() + ";" + GetClangInternalIncludes());
 	master_source.Clear();
 	String editfile = NormalizePath(theide->editfile);
-	if(editfile.GetCount() && IsCHeaderFile(editfile))
-		master_source = FindMasterSource(hdepend, GetIdeWorkspace(), editfile);
+	if(editfile.GetCount() && IsCHeaderFile(editfile)) {
+//		master_source = FindMasterSource(hdepend, GetIdeWorkspace(), editfile);
+		hdepend2.Dirty();
+		hdepend2.SetIncludes(theide->GetCurrentIncludePath() + ";" + GetClangInternalIncludes());
+		master_source = FindMasterSource(hdepend2, GetIdeWorkspace(), editfile);
+		DLOG("Master source " << editfile << " -> " << master_source);
+	}
+	
+	// TODO: Remove
+	hdepend2.WhenBlitzBlock = [=](const String& inc, const String& path) {
+		PutConsole(String() << inc << " blocks BLITZ of " << path);
+	};
+	if(hdepend2.BlitzApproved(editfile))
+		PutConsole(editfile + " BLITZ approved");
 }
 
 bool AssistEditor::DoIncludeTrick(Index<String>& visited, int level, StringBuffer& out, String path, const String& target_path, int& line_delta)
diff --git a/uppsrc/ide/clang/Indexer.cpp b/uppsrc/ide/clang/Indexer.cpp
index 58fcb9633..34bac9895 100644
--- a/uppsrc/ide/clang/Indexer.cpp
+++ b/uppsrc/ide/clang/Indexer.cpp
@@ -58,6 +58,49 @@ String FindMasterSource(Hdepend& hdepend, const Workspace& wspc, const String& h
 	return master_source;
 }
 
+String FindMasterSource(PPInfo& hdepend, const Workspace& wspc, const String& header_file_)
+{
+	String master_source;
+	String header_file = NormalizePath(header_file_);
+	DDUMP(header_file);
+	for(int pass = 0; pass < 2; pass++) { // all packages in second pass
+		VectorMap<String, Time> deps;
+		for(int i = 0; i < wspc.GetCount(); i++) { // find package of included file
+			const Package& pk = wspc.GetPackage(i);
+			String pk_name = wspc[i];
+
+			auto Chk = [&] {
+				for(int i = 0; i < pk.file.GetCount(); i++) {
+					String path = SourcePath(pk_name, pk.file[i]);
+					if(!PathIsEqual(header_file, path) && IsSourceFile(path) && GetFileLength(path) < 200000) {
+						hdepend.GatherDependencies(path, deps);
+						DDUMP(deps);
+						if(deps.Find(header_file) >= 0) {
+							master_source = path;
+							return true;
+						}
+					}
+				}
+				return false;
+			};
+
+			if(pass) { // check all files
+				if(Chk())
+					return master_source;
+			}
+			else
+			for(int i = 0; i < pk.file.GetCount(); i++) { // check files of package
+				if(PathIsEqual(header_file, SourcePath(pk_name, pk.file[i]))) {
+					if(Chk())
+						return master_source;
+					break;
+				}
+			}
+		}
+	}
+	return master_source;
+}
+
 void AnnotationItem::Serialize(Stream& s)
 {
 	s % kind
diff --git a/uppsrc/ide/clang/clang.h b/uppsrc/ide/clang/clang.h
index d44a43fac..70903f704 100644
--- a/uppsrc/ide/clang/clang.h
+++ b/uppsrc/ide/clang/clang.h
@@ -185,6 +185,7 @@ void StartAutoComplete(const CurrentFileContext& ctx, int line, int column, bool
 void CancelAutoComplete();
 
 String FindMasterSource(Hdepend& hdepend, const Workspace& wspc, const String& header_file);
+String FindMasterSource(PPInfo& hdepend, const Workspace& wspc, const String& header_file);
 
 struct FileAnnotation0 {
 	String defines = "<not_loaded>";
diff --git a/uppsrc/ide/clang/todo.txt b/uppsrc/ide/clang/todo.txt
index b1314d1d0..e932f35f5 100644
--- a/uppsrc/ide/clang/todo.txt
+++ b/uppsrc/ide/clang/todo.txt
@@ -103,6 +103,10 @@ ISUES:
 
 - FFound key to reuse pane for repeated finds
 
+- Sort navigator in header order
+
+- mainconfig/dependecies
+
 GREAT CODEBASE PURGE:
 
 - ReferenceDlg
@@ -140,6 +144,8 @@ GREAT CODEBASE PURGE:
 
 - Remove //$
 
+- Reindex all files should clear .ppi cache for hdepends too.
+
 NONCLANG:
 
 - QTF :\1label\1:
@@ -153,12 +159,16 @@ void AppExit__()
 {
 	Thread::ShutdownThreads();
 
+- tree drag/drop does not show texts
+
 
 NTH:
 
+- BLITZ dialog
+
 - reindex source files - add progress
 
-- Use BlitzFile
+- Use BlitzFile function
 
 - autocomplete <Functions>, <macros>, ....