Alena/CodeAnalizer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: Alena:9d864863c6d27f5dab68ea5a0eec61b4679b6833
Branches Tags
Alena:master Alena:v2.1 Alena:main
...
compare: Alena:v2.1
Branches Tags
Alena:v2.1 Alena:master Alena:main

5 Commits
9d864863c6 ... v2.1

Author SHA1 Message Date
Дозорова Алёна
ae1d7a12b8 мини рефакторинг 
добавлены пачноты
 2026-07-24 14:07:34 +04:00
Дозорова Алёна
eb8dac7627 Попытки в мультипарсинг 2026-07-24 13:47:21 +04:00
Дозорова Алёна
b2a13528ee Переехали на графовую модель, засунули все в докер, поменяли парсинг 2026-07-23 08:34:26 +04:00
Дозорова Алёна
c503fc4990 Merge branch 'master' of https://git.mcrt.space/Alena/CodeAnalizer 2026-07-22 22:37:25 +04:00
Дозорова Алёна
b6829ce753 переехали на вектурную бд 2026-07-22 22:36:12 +04:00
27 changed files with 1632 additions and 237 deletions
Expand all files Collapse all files

15
CodeBase/CodeBase.csproj
View File

@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk.Web"> <Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net10.0</TargetFramework> <TargetFramework>net10.0</TargetFramework>
@@ -7,8 +7,21 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.35.1" />
<PackageReference Include="Grpc.Tools" Version="2.83.0">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
<Protobuf Include="scip.proto" GrpcServices="None" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.6" /> <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.6" />
<PackageReference Include="Microsoft.Build.Locator" Version="1.11.2" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="5.6.0" /> <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="5.6.0" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="5.6.0" />
<PackageReference Include="Microsoft.CodeAnalysis.Workspaces.MSBuild" Version="5.6.0" />
<PackageReference Include="Microsoft.Build.Framework" Version="17.11.48" ExcludeAssets="runtime" PrivateAssets="all" />
<PackageReference Include="Neo4j.Driver" Version="6.3.0" />
<PackageReference Include="OpenAI" Version="2.12.0" /> <PackageReference Include="OpenAI" Version="2.12.0" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="10.2.3" /> <PackageReference Include="Swashbuckle.AspNetCore" Version="10.2.3" />
</ItemGroup> </ItemGroup>

16
CodeBase/Controllers/CodeController.cs
View File

@@ -8,19 +8,21 @@ namespace CodeBase.Controllers
{ {
[Route("api/[controller]")] [Route("api/[controller]")]
[ApiController] [ApiController]
public class CodeController(CodeService service) : ControllerBase public class CodeController(CodeService service,
ScipProcessingService scipProcessingService) : ControllerBase
{ {
[HttpPost("analyze")]
public async Task<CodeChunk> AnalyzeRepository(string path, string name)
{
var chunks = await service.GetCodeChunksAsync(path, name);
return chunks[0];
}
[HttpPost("answer")] [HttpPost("answer")]
public async Task<string> GetAnswer(string question, string name) public async Task<string> GetAnswer(string question, string name)
{ {
var answer = await service.GetAnswerAsync(name, question); var answer = await service.GetAnswerAsync(name, question);
return answer; return answer;
} }
[HttpPost("parser")]
public async Task<string> ParserProject(string path, string name, Language language)
{
await scipProcessingService.ProcessAndSaveProjectAsync(path, language.ToString(), name);
return "Проект добавлен в базу";
}
} }
} }

12
CodeBase/Models/CodeChunk.cs
View File

@@ -2,11 +2,13 @@
{ {
public class CodeChunk public class CodeChunk
{ {
public string Id { get; set; } // Уникальный идентификатор из SCIP
public string ProjectName { get; set; }
public string EntityName { get; set; } // Человекочитаемое имя (например, ProcessData)
public string FilePath { get; set; } public string FilePath { get; set; }
public string ClassName { get; set; } public string Content { get; set; } // Вырезанный сырой исходный код
public string MethodName { get; set; } public string Language { get; set; }
public string Documentation { get; set; } public float[] Embedding { get; set; } // Вектор из GraphCodeBERT
public string Content { get; set; } public List<string> OutgoingCalls { get; set; } = new();
public float[] Vector { get; set; }
} }
} }

14
CodeBase/Models/Enums.cs Normal file
View File

@@ -0,0 +1,14 @@
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
namespace CodeBase.Models
{
[JsonConverter(typeof(StringEnumConverter))]
public enum Language
{
csharp,
python,
typescript,
go
}
}

16
CodeBase/Models/RetrievedContext.cs Normal file
View File

@@ -0,0 +1,16 @@
namespace CodeBase.Models
{
public class RetrievedContext
{
public string EntityName { get; set; }
public string FilePath { get; set; }
public string Content { get; set; }
public string ProjectName { get; set; }
// Оценка релевантности (полезно для отладки качества поиска)
public double SimilarityScore { get; set; }
public List<string> OutgoingDependencies { get; set; } = new();
public List<string> IncomingDependencies { get; set; } = new();
}
}

85
CodeBase/Orchestrators/ScipOrchestrator.cs Normal file
View File

@@ -0,0 +1,85 @@
using System.Diagnostics;
namespace CodeBase.Orchestrators
{
public class ScipOrchestrator
{
private readonly Dictionary<string, string> _indexerCommands;
// Внедряем IConfiguration через конструктор
public ScipOrchestrator(IConfiguration configuration)
{
// Считываем секцию из appsettings.json в словарь при старте приложения
_indexerCommands = configuration.GetSection("ScipIndexers").Get<Dictionary<string, string>>()
?? new Dictionary<string, string>();
}
public async Task<bool> GenerateScipAsync(string localRepoPath, string language)
{
var absolutePath = Path.GetFullPath(localRepoPath);
var scipFilePath = Path.Combine(absolutePath, "index.scip");
if (File.Exists(scipFilePath))
{
Console.WriteLine($"[SCIP] Файл графа уже существует: {scipFilePath}");
return true;
}
var langKey = language.Trim().ToLowerInvariant();
// 1. Ищем команду по ключу из файла
if (!_indexerCommands.TryGetValue(langKey, out var commandTemplate))
{
Console.WriteLine($"[SCIP ERROR] Язык '{langKey}' не найден в конфигурации appsettings.json.");
return false;
}
// Обманываем scip-python, подкидывая ему маркер корня проекта, чтобы он не требовал Git
if (langKey == "python")
{
var dummyFilePath = Path.Combine(absolutePath, "pyproject.toml");
var setupFilePath = Path.Combine(absolutePath, "setup.py");
// Если ни одного из файлов конфигурации нет, создаем пустышку
if (!File.Exists(dummyFilePath) && !File.Exists(setupFilePath))
{
File.WriteAllText(dummyFilePath, ""); // Создаем физический файл нулевого размера
Console.WriteLine("[SCIP] Создан пустой файл pyproject.toml для обхода ограничений парсера.");
}
}
// 2. Подставляем путь к репозиторию в шаблон команды (заменяем {0})
string dockerArguments = string.Format(commandTemplate, absolutePath);
Console.WriteLine($"[SCIP] Запускаем индексацию для проекта: {absolutePath} (Язык: {langKey})");
var processStartInfo = new ProcessStartInfo
{
FileName = "docker",
Arguments = dockerArguments,
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
};
using var process = new Process { StartInfo = processStartInfo };
process.Start();
var outputTask = process.StandardOutput.ReadToEndAsync();
var errorTask = process.StandardError.ReadToEndAsync();
await process.WaitForExitAsync();
if (process.ExitCode != 0)
{
var error = await errorTask;
Console.WriteLine($"[SCIP ERROR] Ошибка генерации графа:\n{error}");
return false;
}
return true;
}
}
}

41
CodeBase/Program.cs
View File

@@ -1,22 +1,53 @@
using CodeBase.Orchestrators;
using CodeBase.Parsers;
using CodeBase.Repositories;
using CodeBase.Services; using CodeBase.Services;
using CodeBase.Warehouse; using Microsoft.Build.Locator;
using Neo4j.Driver;
using System.Text.Json.Serialization;
if (!MSBuildLocator.IsRegistered)
{
MSBuildLocator.RegisterDefaults();
}
var builder = WebApplication.CreateBuilder(args); var builder = WebApplication.CreateBuilder(args);
// Add services to the container. builder.Services.AddSingleton<DatabaseInitializer>();
builder.Services.AddSingleton<ChunkWarehouse>(); // Add services to the container.
builder.Services.AddSingleton<IDriver>(sp =>
GraphDatabase.Driver(
"bolt://localhost:7687",
AuthTokens.Basic("neo4j", "password123")
)
);
builder.Services.AddSingleton<ScipOrchestrator>();
builder.Services.AddTransient<ScipParser>();
builder.Services.AddTransient<ScipProcessingService>();
builder.Services.AddTransient<VectorizationService>(); builder.Services.AddTransient<VectorizationService>();
builder.Services.AddTransient<LlmService>(); builder.Services.AddTransient<LlmService>();
builder.Services.AddScoped<LlmPromptBuilder>(); builder.Services.AddScoped<LlmPromptBuilder>();
builder.Services.AddTransient<ChunkService>();
builder.Services.AddTransient<CodeService>(); builder.Services.AddTransient<CodeService>();
builder.Services.AddTransient<ScipRepository>();
builder.Services.AddControllers(); builder.Services.AddControllers().AddJsonOptions(options =>
{
// Добавляем конвертер строковых енумов глобально
options.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter());
}); ;
builder.Services.AddSwaggerGen(); builder.Services.AddSwaggerGen();
var app = builder.Build(); var app = builder.Build();
using (var scope = app.Services.CreateScope())
{
var dbInit = scope.ServiceProvider.GetRequiredService<DatabaseInitializer>();
await dbInit.InitializeAsync();
}
// Configure the HTTP request pipeline. // Configure the HTTP request pipeline.
if (app.Environment.IsDevelopment()) if (app.Environment.IsDevelopment())
{ {

45
CodeBase/Repositories/DatabaseInitializer.cs Normal file
View File

@@ -0,0 +1,45 @@
using Neo4j.Driver;
namespace CodeBase.Repositories
{
public class DatabaseInitializer
{
private readonly IDriver _neo4jDriver;
public DatabaseInitializer(IDriver neo4jDriver)
{
_neo4jDriver = neo4jDriver;
}
public async Task InitializeAsync()
{
try
{
await using var session = _neo4jDriver.AsyncSession();
// IF NOT EXISTS гарантирует, что запрос не упадет с ошибкой,
// если индекс уже был создан при предыдущем запуске
var query = @"
CREATE VECTOR INDEX code_embeddings IF NOT EXISTS
FOR (m:CodeEntity) ON (m.embedding)
OPTIONS {
indexConfig: {
`vector.dimensions`: 768, // <-- УКАЖИ ТУТ РАЗМЕРНОСТЬ ТВОЕЙ МОДЕЛИ
`vector.similarity_function`: 'cosine'
}
}";
await session.ExecuteWriteAsync(async tx =>
{
await tx.RunAsync(query);
});
Console.WriteLine("[БД] Векторный индекс успешно инициализирован.");
}
catch (Exception ex)
{
Console.WriteLine($"[БД ОШИБКА] Ошибка при создании индекса: {ex.Message}");
}
}
}
}

96
CodeBase/Repositories/ScipRepository.cs Normal file
View File

@@ -0,0 +1,96 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using CodeBase.Models;
using Neo4j.Driver;
namespace CodeBase.Repositories
{
public class ScipRepository
{
private readonly IDriver _neo4jDriver;
public ScipRepository(IDriver neo4jDriver)
{
_neo4jDriver = neo4jDriver;
}
public async Task SaveChunksAsync(List<CodeChunk> chunks)
{
if (chunks == null || chunks.Count == 0) return;
await using var session = _neo4jDriver.AsyncSession();
await session.ExecuteWriteAsync(async tx =>
{
foreach (var chunk in chunks)
{
var uniqueId = $"{chunk.ProjectName}::{chunk.Id}";
var query = @"
MERGE (p:Project {name: $projectName})
MERGE (m:CodeEntity {id: $id})
SET m.projectName = $projectName,
m.name = $name,
m.filePath = $filePath,
m.code = $code,
m.language = $language,
m.embedding = $embedding
MERGE (p)-[:CONTAINS]->(m)";
await tx.RunAsync(query, new
{
id = uniqueId,
projectName = chunk.ProjectName,
name = chunk.EntityName,
filePath = chunk.FilePath,
code = chunk.Content,
language = chunk.Language,
embedding = chunk.Embedding
});
}
});
}
/// <summary>
/// Ищет в графе узлы, наиболее близкие к переданному вектору.
/// </summary>
/// <param name="queryVector">Вектор вопроса пользователя</param>
/// <param name="projectName">Имя проекта для фильтрации (опционально)</param>
/// <param name="topK">Сколько кусков кода вернуть</param>
public async Task<List<RetrievedContext>> FindSimilarNodesAsync(float[] queryVector, string projectName = null, int topK = 5)
{
await using var session = _neo4jDriver.AsyncSession();
var result = await session.ExecuteReadAsync(async tx =>
{
// Базовый запрос к векторному индексу
string cypherQuery = @"
CALL db.index.vector.queryNodes('code_embeddings', $topK, $queryVector)
YIELD node AS method, score
WHERE $projectName IS NULL OR method.projectName = $projectName
RETURN method.projectName AS Project, method.name AS Name, method.filePath AS Path, method.code AS Code, score
ORDER BY score DESC";
var cursor = await tx.RunAsync(cypherQuery, new { topK, queryVector, projectName });
var contexts = new List<RetrievedContext>();
while (await cursor.FetchAsync())
{
contexts.Add(new RetrievedContext
{
ProjectName = cursor.Current["Project"].As<string>(), // Читаем имя проекта
EntityName = cursor.Current["Name"].As<string>(),
FilePath = cursor.Current["Path"].As<string>(),
Content = cursor.Current["Code"].As<string>(),
SimilarityScore = cursor.Current["score"].As<double>()
});
}
return contexts;
});
return result;
}
}
}

62
CodeBase/Services/ChunkService.cs
View File

@@ -1,62 +0,0 @@
using CodeBase.Models;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.CSharp.Syntax;
namespace CodeBase.Services
{
public class ChunkService(VectorizationService vectorizationService)
{
public List<CodeChunk> ChunkCSharpFile(string filePath, string fileContent)
{
var chunks = new List<CodeChunk>();
var syntaxTree = CSharpSyntaxTree.ParseText(fileContent);
var root = syntaxTree.GetRoot();
// 1. Собираем методы (как и раньше)
var methods = root.DescendantNodes().OfType<MethodDeclarationSyntax>();
foreach (var method in methods)
{
chunks.Add(new CodeChunk
{
FilePath = filePath,
MethodName = method.Identifier.Text,
Content = method.ToFullString().Trim()
});
}
// 2. ДОБАВЛЯЕМ СБОР ENUM (Перечислений)
var enums = root.DescendantNodes().OfType<EnumDeclarationSyntax>();
foreach (var enumSyntax in enums)
{
chunks.Add(new CodeChunk
{
FilePath = filePath,
MethodName = enumSyntax.Identifier.Text, // Сохраняем имя енама
Content = enumSyntax.ToFullString().Trim()
});
}
// 3. ДОБАВЛЯЕМ СБОР КЛАССОВ-МОДЕЛЕЙ (без методов)
var classes = root.DescendantNodes().OfType<ClassDeclarationSyntax>();
foreach (var classSyntax in classes)
{
// Берем только свойства, чтобы понимать структуру модели
var properties = classSyntax.Members.OfType<PropertyDeclarationSyntax>();
if (properties.Any())
{
chunks.Add(new CodeChunk
{
FilePath = filePath,
MethodName = classSyntax.Identifier.Text,
// Сохраняем объявление класса и его свойства
Content = $"class {classSyntax.Identifier.Text} {{\n" +
string.Join("\n", properties.Select(p => p.ToFullString().Trim())) +
"\n}"
});
}
}
return chunks;
}
}
}

84
CodeBase/Services/CodeService.cs
View File

@@ -1,102 +1,32 @@
using CodeBase.Models; using CodeBase.Models;
using CodeBase.Warehouse; using CodeBase.Repositories;
namespace CodeBase.Services namespace CodeBase.Services
{ {
public class CodeService(ChunkService service, public class CodeService(ScipRepository repository,
ChunkWarehouse warehouse,
VectorizationService vectorizationService, VectorizationService vectorizationService,
LlmService llmService) LlmService llmService)
{ {
public async Task<List<CodeChunk>> GetCodeChunksAsync(string path, string name)
{
if (!Path.Exists(path))
{
throw new Exception("Путь не найден");
}
var allChunks = new List<CodeChunk>();
// Находим все .cs файлы во всех вложенных папках
var csFiles = Directory.GetFiles(path, "*.cs", SearchOption.AllDirectories);
foreach (var filePath in csFiles)
{
var content = File.ReadAllText(filePath);
// Парсим каждый файл отдельно, сохраняя его оригинальный путь
var fileChunks = service.ChunkCSharpFile(filePath, content);
allChunks.AddRange(fileChunks);
}
var chunks = await vectorizationService.EnrichChunksWithVectorsAsync(allChunks);
warehouse.UpsertDictionary(name, allChunks);
return chunks;
}
public async Task<string> GetAnswerAsync(string name, string question) public async Task<string> GetAnswerAsync(string name, string question)
{ {
var query = await vectorizationService.GetVectorAsync(question); var query = await vectorizationService.GetVectorAsync(question);
var answerVectors = Search(query, name, 15); var answerVectors = await SearchAsync(query, name, 15);
var answer = await llmService.AskQuestionAsync(question, answerVectors.Select(s => s.Chunk).ToList()); var answer = await llmService.AskQuestionAsync(question, answerVectors);
return answer; return answer;
} }
// Главный метод поиска // Главный метод поиска
public List<(CodeChunk Chunk, float Score)> Search( public async Task<List<RetrievedContext>> SearchAsync(
float[] queryVector, float[] queryVector,
string name, string name,
int topK = 3) // Возвращаем топ-3 результата int topK = 3) // Возвращаем топ-3 результата
{ {
var memoryBase = warehouse.GetChunks(name); var results = await repository.FindSimilarNodesAsync(queryVector, name, topK);
var results = new List<(CodeChunk, float)>(); return results;
foreach (var chunk in memoryBase)
{
if (chunk.Vector == null || chunk.Vector.Length == 0)
continue;
// Считаем близость вектора вопроса к вектору кода
float similarity = CalculateCosineSimilarity(queryVector, chunk.Vector);
results.Add((chunk, similarity));
}
// Сортируем по убыванию сходства (чем ближе к 1, тем лучше)
return results
.OrderByDescending(x => x.Item2)
.Take(topK)
.ToList();
}
// Математика косинусного сходства
private float CalculateCosineSimilarity(float[] vectorA, float[] vectorB)
{
if (vectorA.Length != vectorB.Length)
throw new ArgumentException("Векторы должны быть одинаковой длины (например, 768).");
float dotProduct = 0;
float magnitudeA = 0;
float magnitudeB = 0;
for (int i = 0; i < vectorA.Length; i++)
{
dotProduct += vectorA[i] * vectorB[i];
magnitudeA += vectorA[i] * vectorA[i];
magnitudeB += vectorB[i] * vectorB[i];
}
// Защита от деления на ноль
if (magnitudeA == 0 || magnitudeB == 0)
return 0;
return (float)(dotProduct / (Math.Sqrt(magnitudeA) * Math.Sqrt(magnitudeB)));
} }
} }
} }

59
CodeBase/Services/LlmPromptBuilder.cs
View File

@@ -4,38 +4,49 @@ using System.Text;
public class LlmPromptBuilder public class LlmPromptBuilder
{ {
public string BuildPrompt(string userQuestion, List<CodeChunk> foundChunks) public string BuildPrompt(string userQuestion, List<RetrievedContext> graphContexts)
{ {
var sb = new StringBuilder(); var promptBuilder = new StringBuilder();
// 1. Задаем жесткую роль и ограничения // Универсальная роль архитектора без привязки к конкретному языку
sb.AppendLine("Ты — опытный разработчик и архитектор. Твоя задача — ответить на вопрос пользователя."); promptBuilder.AppendLine("Ты — опытный ИТ-архитектор и разработчик. Твоя задача — ответить на вопрос пользователя, опираясь ИСКЛЮЧИТЕЛЬНО на предоставленный граф вызовов и зависимостей кода. Не придумывай методы или классы, которых нет в контексте.");
sb.AppendLine("ОТВЕЧАЙ СТРОГО НА ОСНОВЕ ПРЕДОСТАВЛЕННОГО КОДА НИЖЕ."); promptBuilder.AppendLine("\nКонтекст из кодовой базы (Граф зависимостей):");
sb.AppendLine("Если в коде нет ответа на вопрос, честно скажи: «В данном фрагменте кода нет этой информации». Не придумывай функции, которых нет в тексте.");
sb.AppendLine("\n================ ПРЕДОСТАВЛЕННЫЙ КОД ================"); foreach (var node in graphContexts)
// 2. Вклеиваем найденные чанки с контекстом
foreach (var chunk in foundChunks)
{ {
sb.AppendLine($"Файл: {chunk.FilePath}"); promptBuilder.AppendLine("--------------------------------------------------");
sb.AppendLine($"Класс: {chunk.ClassName}"); promptBuilder.AppendLine($"[УЗЕЛ ГРАФА]");
sb.AppendLine($"Метод: {chunk.MethodName}"); promptBuilder.AppendLine($"Проект: {node.ProjectName}");
if (!string.IsNullOrEmpty(chunk.Documentation)) promptBuilder.AppendLine($"Имя сущности: {node.EntityName}");
promptBuilder.AppendLine($"Файл: {node.FilePath}");
// Добавляем зависимости ВНИЗ (что использует метод)
if (node.OutgoingDependencies != null && node.OutgoingDependencies.Any())
{ {
sb.AppendLine($"Документация: {chunk.Documentation}"); promptBuilder.AppendLine("\n[ИСПОЛЬЗУЕТ ВНУТРИ СЕБЯ]:");
foreach (var dep in node.OutgoingDependencies.Distinct())
{
promptBuilder.AppendLine($"- {dep}");
}
} }
sb.AppendLine("Код:");
sb.AppendLine("```csharp"); // Добавляем зависимости ВВЕРХ (кто зависит от метода)
sb.AppendLine(chunk.Content); if (node.IncomingDependencies != null && node.IncomingDependencies.Any())
sb.AppendLine("```"); {
sb.AppendLine("--------------------------------------------------"); promptBuilder.AppendLine("\n[ВЫЗЫВАЕТСЯ ИЗ]:");
foreach (var caller in node.IncomingDependencies.Distinct())
{
promptBuilder.AppendLine($"- {caller}");
}
}
promptBuilder.AppendLine("\nКод:");
promptBuilder.AppendLine(node.Content);
} }
// 3. Добавляем сам вопрос promptBuilder.AppendLine("--------------------------------------------------");
sb.AppendLine("\n================ ВОПРОС ПОЛЬЗОВАТЕЛЯ ================"); promptBuilder.AppendLine($"Вопрос пользователя: {userQuestion}");
sb.AppendLine(userQuestion);
return sb.ToString(); return promptBuilder.ToString();
} }
} }

5
CodeBase/Services/LlmService.cs
View File

@@ -4,6 +4,7 @@ using OpenAI.Chat;
using System; using System;
using System.ClientModel; using System.ClientModel;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Xml.Linq;
public class LlmService public class LlmService
{ {
@@ -30,11 +31,11 @@ public class LlmService
_builder = llmPromptBuilder; _builder = llmPromptBuilder;
} }
public async Task<String> AskQuestionAsync(string userQuestion, List<CodeChunk> foundChunks) public async Task<String> AskQuestionAsync(string userQuestion, List<RetrievedContext> grafContext)
{ {
try try
{ {
var finalPrompt = _builder.BuildPrompt(userQuestion, foundChunks); var finalPrompt = _builder.BuildPrompt(userQuestion, grafContext);
// Формируем сообщение. Поскольку мы уже зашили роль и инструкции // Формируем сообщение. Поскольку мы уже зашили роль и инструкции
// внутрь finalPrompt с помощью LlmPromptBuilder, // внутрь finalPrompt с помощью LlmPromptBuilder,
// передаем все это как UserMessage. // передаем все это как UserMessage.

75
CodeBase/Services/ScipParser.cs Normal file
View File

@@ -0,0 +1,75 @@
using CodeBase.Models;
using CodeBase.Orchestrators;
using CodeBase.Services;
using Google.Protobuf;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace CodeBase.Parsers
{
public class ScipParser(ScipOrchestrator scipOrchestrator)
{
public async Task<List<CodeChunk>> ParseProjectAsync(string projectRootPath, string language, string name)
{
var scipFilePath = Path.Combine(projectRootPath, "index.scip");
if (!File.Exists(scipFilePath))
{
await scipOrchestrator.GenerateScipAsync(projectRootPath, language);
}
var filter = new UniversalFileFilter();
using var stream = File.OpenRead(scipFilePath);
var scipIndex = Scip.Index.Parser.ParseFrom(stream);
var extractedChunks = new List<CodeChunk>();
foreach (var document in scipIndex.Documents)
{
if (!filter.IsValidCodeFile(document.RelativePath)) continue;
var absoluteFilePath = Path.Combine(projectRootPath, document.RelativePath);
if (!File.Exists(absoluteFilePath)) continue;
var fileLines = await File.ReadAllLinesAsync(absoluteFilePath);
var declarations = document.Occurrences.Where(o => (o.SymbolRoles & 1) == 1);
foreach (var occ in declarations)
{
int startLine = occ.Range[0];
int endLine = occ.Range.Count == 3 ? occ.Range[0] : occ.Range[2];
if (startLine < 0 || endLine >= fileLines.Length) continue;
var codeSnippet = string.Join(
Environment.NewLine,
fileLines.Skip(startLine).Take(endLine - startLine + 1)
);
// Возвращаем чанки пока БЕЗ векторов
extractedChunks.Add(new CodeChunk
{
Id = occ.Symbol,
EntityName = ExtractSimpleName(occ.Symbol),
FilePath = document.RelativePath,
Content = codeSnippet,
Language = document.Language,
ProjectName = name
});
}
}
return extractedChunks;
}
private string ExtractSimpleName(string scipSymbol)
{
if (string.IsNullOrEmpty(scipSymbol)) return "Unknown";
var parts = scipSymbol.Split(new[] { '#', '.', '(', ')' }, StringSplitOptions.RemoveEmptyEntries);
return parts.LastOrDefault() ?? scipSymbol;
}
}
}

49
CodeBase/Services/ScipProcessingService.cs Normal file
View File

@@ -0,0 +1,49 @@
using System;
using System.Threading.Tasks;
using CodeBase.Parsers;
using CodeBase.Repositories;
namespace CodeBase.Services
{
public class ScipProcessingService
{
private readonly ScipParser _parser;
private readonly VectorizationService _vectorizationService;
private readonly ScipRepository _repository;
// Внедрение зависимостей
public ScipProcessingService(
ScipParser parser,
VectorizationService vectorizationService,
ScipRepository repository)
{
_parser = parser;
_vectorizationService = vectorizationService;
_repository = repository;
}
public async Task ProcessAndSaveProjectAsync(string projectRootPath, string lang, string name)
{
try
{
Console.WriteLine($"[СЕРВИС] Запуск парсинга проекта: {projectRootPath}");
// 1. Получаем "сырые" чанки из парсера
var chunks = await _parser.ParseProjectAsync(projectRootPath, lang, name);
Console.WriteLine($"[СЕРВИС] Найдено {chunks.Count} сущностей. Начинаем векторизацию...");
// 2. Обогащаем каждый чанк векторным представлением GraphCodeBERT
chunks = await _vectorizationService.EnrichChunksWithVectorsAsync(chunks);
// 3. Сохраняем готовую сборку в графовую базу
await _repository.SaveChunksAsync(chunks);
Console.WriteLine("[СЕРВИС] Проект успешно обработан и сохранен в БД.");
}
catch (Exception ex)
{
Console.WriteLine($"[СЕРВИС ОШИБКА] Сбой при обработке проекта: {ex.Message}");
}
}
}
}

52
CodeBase/Services/UniversalFileFilter.cs Normal file
View File

@@ -0,0 +1,52 @@
namespace CodeBase.Services
{
public class UniversalFileFilter
{
// 1. Оставляем только те языки, которые нам реально интересны
private readonly HashSet<string> _allowedExtensions = new(StringComparer.OrdinalIgnoreCase)
{
".cs", ".py", ".go", ".ts", ".js", ".java", ".cpp", ".c", ".h", ".json"
};
// 2. Глобальные папки с мусором и зависимостями (универсально для разных стеков)
private readonly string[] _ignoredDirectories =
{
"/obj/", "\\obj\\",
"/bin/", "\\bin\\",
"/node_modules/", "\\node_modules\\",
"/venv/", "\\venv\\",
"/.env/", "\\.env\\",
"/dist/", "\\dist\\",
"/build/", "\\build\\",
"/.git/", "\\.git\\"
};
// 3. Паттерны автосгенерированных файлов
private readonly string[] _ignoredFileSuffixes =
{
".g.cs",
".designer.cs",
".generated.cs",
"AssemblyInfo.cs"
};
public bool IsValidCodeFile(string relativePath)
{
if (string.IsNullOrWhiteSpace(relativePath)) return false;
// Проверка 1: Расширение файла
var ext = Path.GetExtension(relativePath);
if (!_allowedExtensions.Contains(ext)) return false;
// Проверка 2: Находится ли файл в мусорной папке
if (_ignoredDirectories.Any(dir => relativePath.Contains(dir, StringComparison.OrdinalIgnoreCase)))
return false;
// Проверка 3: Является ли файл автосгенерированным
if (_ignoredFileSuffixes.Any(suffix => relativePath.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)))
return false;
return true;
}
}
}

30
CodeBase/Services/VectorizationService.cs
View File

@@ -22,6 +22,28 @@ namespace CodeBase.Services
// Адрес нашего локального Python-сервиса // Адрес нашего локального Python-сервиса
_httpClient.BaseAddress = new Uri("http://localhost:8000/"); _httpClient.BaseAddress = new Uri("http://localhost:8000/");
} }
private const int MaxCodeLength = 1500;
private string BuildSafeContext(CodeChunk chunk)
{
// Метаданные (оставляем всегда целиком)
string metadata = $"File: {chunk.FilePath}\nEntity: {chunk.EntityName}\nCode:\n";
// Вычисляем, сколько символов у нас осталось для самого кода
int remainingLength = MaxCodeLength - metadata.Length;
string code = chunk.Content;
// Если код слишком длинный — аккуратно отрезаем хвост
if (code.Length > remainingLength && remainingLength > 0)
{
code = code.Substring(0, remainingLength) + "\n...[TRUNCATED]";
Console.WriteLine($"[ВЕКТОРИЗАЦИЯ] Метод {chunk.EntityName} слишком длинный. Обрезан до {MaxCodeLength} символов.");
}
return metadata + code;
}
public async Task<float[]> GetVectorAsync(string text) public async Task<float[]> GetVectorAsync(string text)
@@ -55,7 +77,7 @@ namespace CodeBase.Services
{ {
// 1. Склеиваем контекст // 1. Склеиваем контекст
// Мы даем нейросети подсказку о том, где именно лежит этот код // Мы даем нейросети подсказку о том, где именно лежит этот код
string contextText = $"File: {chunk.FilePath}\nClass: {chunk.ClassName}\nMethod: {chunk.MethodName}\nCode:\n{chunk.Content}"; string contextText = BuildSafeContext(chunk);
var requestBody = new VectorizeRequest { text = contextText }; var requestBody = new VectorizeRequest { text = contextText };
@@ -71,13 +93,13 @@ namespace CodeBase.Services
if (result != null && result.vector != null) if (result != null && result.vector != null)
{ {
// 4. Сохраняем вектор прямо в наш объект в памяти // 4. Сохраняем вектор прямо в наш объект в памяти
chunk.Vector = result.vector; chunk.Embedding = result.vector;
Console.WriteLine($"[+] Векторизован метод: {chunk.MethodName}"); Console.WriteLine($"[+] Векторизована сущность: {chunk.EntityName}");
} }
} }
else else
{ {
Console.WriteLine($"[-] Ошибка API для {chunk.MethodName}: {response.StatusCode}"); Console.WriteLine($"[-] Ошибка API для {chunk.EntityName}: {response.StatusCode}");
} }
} }

42
CodeBase/Warehouse/ChunkWarehouse.cs
View File

@@ -1,42 +0,0 @@
using CodeBase.Models;
namespace CodeBase.Warehouse
{
public class ChunkWarehouse
{
public Dictionary<string, List<CodeChunk>> dictionary;
public ChunkWarehouse()
{
this.dictionary = new();
}
public void UpsertDictionary(string name, List<CodeChunk> chunks)
{
if (dictionary.ContainsKey(name))
{
dictionary[name] = chunks;
}
else dictionary.TryAdd(name, chunks);
}
public void DeleteElement(string name)
{
dictionary.Remove(name);
}
public List<CodeChunk> GetChunks(string name)
{
if(dictionary.TryGetValue(name, out var chunks))
{
return chunks;
}
throw new Exception("Такого проекта нет");
}
public void ClearDictionary()
{
dictionary.Clear();
}
}
}

18
CodeBase/appsettings.json
View File

@@ -1,9 +1,15 @@
{ {
"Logging": { "Logging": {
"LogLevel": { "LogLevel": {
"Default": "Information", "Default": "Information",
"Microsoft.AspNetCore": "Warning" "Microsoft.AspNetCore": "Warning"
}
},
"AllowedHosts": "*",
"ScipIndexers": {
"go": "run --rm -v \"{0}:/workspace\" -w /workspace sourcegraph/scip-go",
"python": "run --rm -v \"{0}:/workspace\" -w /workspace sourcegraph/scip-python scip-python index .",
"csharp": "run --rm -v \"{0}:/workspace\" -w /workspace sourcegraph/scip-dotnet scip-dotnet index",
"typescript": "run --rm -v \"{0}:/workspace\" sourcegraph/scip-typescript index ."
} }
},
"AllowedHosts": "*"
} }

BIN
CodeBase/index.scip Normal file
View File

Binary file not shown.

962
CodeBase/scip.proto Normal file
View File

@@ -0,0 +1,962 @@
// An index contains one or more pieces of information about a given piece of
// source code or software artifact. Complementary information can be merged
// together from multiple sources to provide a unified code intelligence
// experience.
//
// Programs producing a file of this format is an "indexer" and may operate
// somewhere on the spectrum between precision, such as indexes produced by
// compiler-backed indexers, and heurstics, such as indexes produced by local
// syntax-directed analysis for scope rules.
syntax = "proto3";
package scip;
option go_package = "github.com/scip-code/scip/bindings/go/scip/";
option java_multiple_files = true;
option java_outer_classname = "ScipProto";
option java_package = "org.scip_code.scip";
// Index represents a complete SCIP index for a workspace this is rooted at a
// single directory. An Index message payload can have a large memory footprint
// and it's therefore recommended to emit and consume an Index payload one field
// value at a time. To permit streaming consumption of an Index payload, the
// `metadata` field must appear at the start of the stream and must only appear
// once in the stream. Other field values may appear in any order.
message Index {
// Metadata about this index.
Metadata metadata = 1;
// Documents that belong to this index.
repeated Document documents = 2;
// (optional) Symbols that are referenced from this index but are defined in
// an external package (a separate `Index` message). Leave this field empty
// if you assume the external package will get indexed separately. If the
// external package won't get indexed for some reason then you can use this
// field to provide hover documentation for those external symbols.
repeated SymbolInformation external_symbols = 3;
// IMPORTANT: When adding a new field to `Index` here, add a matching
// function in `IndexVisitor` and update `ParseStreaming`.
}
message Metadata {
// Which version of this protocol was used to generate this index?
ProtocolVersion version = 1;
// Information about the tool that produced this index.
ToolInfo tool_info = 2;
// URI-encoded absolute path to the root directory of this index. All
// documents in this index must appear in a subdirectory of this root
// directory.
string project_root = 3;
// Text encoding of the source files on disk that are referenced from
// `Document.relative_path`. This value is unrelated to the `Document.text`
// field, which is a Protobuf string and hence must be UTF-8 encoded.
TextEncoding text_document_encoding = 4;
}
enum ProtocolVersion {
UnspecifiedProtocolVersion = 0;
}
enum TextEncoding {
UnspecifiedTextEncoding = 0;
UTF8 = 1;
UTF16 = 2;
}
message ToolInfo {
// Name of the indexer that produced this index.
string name = 1;
// Version of the indexer that produced this index.
string version = 2;
// Command-line arguments that were used to invoke this indexer.
repeated string arguments = 3;
}
// Document defines the metadata about a source file on disk.
message Document {
// The string ID for the programming language this file is written in.
// The `Language` enum contains the names of most common programming languages.
// This field is typed as a string to permit any programming language, including
// ones that are not specified by the `Language` enum.
string language = 4;
// (Required) Unique path to the text document.
//
// 1. The path must be relative to the directory supplied in the associated
// `Metadata.project_root`.
// 2. The path must not begin with a leading '/'.
// 3. The path must point to a regular file, not a symbolic link.
// 4. The path must use '/' as the separator, including on Windows.
// 5. The path must be canonical; it cannot include empty components ('//'),
// or '.' or '..'.
string relative_path = 1;
// Occurrences that appear in this file.
repeated Occurrence occurrences = 2;
// Symbols that are "defined" within this document.
//
// This should include symbols which technically do not have any definition,
// but have a reference and are defined by some other symbol (see
// Relationship.is_definition).
repeated SymbolInformation symbols = 3;
// (optional) Text contents of this document. Indexers are not expected to
// include the text by default. It's preferable that clients read the text
// contents from the file system by resolving the absolute path from joining
// `Index.metadata.project_root` and `Document.relative_path`. This field
// can be useful for testing or when working with virtual/in-memory documents.
string text = 5;
// Specifies the encoding used for source ranges in this Document.
//
// Usually, this will match the type used to index the string type
// in the indexer's implementation language in O(1) time.
// - For an indexer implemented in JVM/.NET language or JavaScript/TypeScript,
// use UTF16CodeUnitOffsetFromLineStart.
// - For an indexer implemented in Python,
// use UTF32CodeUnitOffsetFromLineStart.
// - For an indexer implemented in Go, Rust or C++,
// use UTF8ByteOffsetFromLineStart.
PositionEncoding position_encoding = 6;
}
// Encoding used to interpret the 'character' value in source ranges.
enum PositionEncoding {
// Default value. This value should not be used by new SCIP indexers
// so that a consumer can process the SCIP index without ambiguity.
UnspecifiedPositionEncoding = 0;
// The 'character' value is interpreted as an offset in terms
// of UTF-8 code units (i.e. bytes).
//
// Example: For the string "🚀 Woo" in UTF-8, the bytes are
// [240, 159, 154, 128, 32, 87, 111, 111], so the offset for 'W'
// would be 5.
UTF8CodeUnitOffsetFromLineStart = 1;
// The 'character' value is interpreted as an offset in terms
// of UTF-16 code units (each is 2 bytes).
//
// Example: For the string "🚀 Woo", the UTF-16 code units are
// ['\ud83d', '\ude80', ' ', 'W', 'o', 'o'], so the offset for 'W'
// would be 3.
UTF16CodeUnitOffsetFromLineStart = 2;
// The 'character' value is interpreted as an offset in terms
// of UTF-32 code units (each is 4 bytes).
//
// Example: For the string "🚀 Woo", the UTF-32 code units are
// ['🚀', ' ', 'W', 'o', 'o'], so the offset for 'W' would be 2.
UTF32CodeUnitOffsetFromLineStart = 3;
}
// Symbol is similar to a URI, it identifies a class, method, or a local
// variable. `SymbolInformation` contains rich metadata about symbols such as
// the docstring.
//
// Symbol has a standardized string representation, which can be used
// interchangeably with `Symbol`. The syntax for Symbol is the following:
// ```
// # (<x>)+ stands for one or more repetitions of <x>
// # (<x>)? stands for zero or one occurrence of <x>
// <symbol> ::= <scheme> ' ' <package> ' ' (<descriptor>)+ | 'local ' <local-id>
// <package> ::= <manager> ' ' <package-name> ' ' <version>
// <scheme> ::= any UTF-8, escape spaces with double space. Must not be empty nor start with 'local'
// <manager> ::= any UTF-8, escape spaces with double space. Use the placeholder '.' to indicate an empty value
// <package-name> ::= same as above
// <version> ::= same as above
// <descriptor> ::= <namespace> | <type> | <term> | <method> | <type-parameter> | <parameter> | <meta> | <macro>
// <namespace> ::= <name> '/'
// <type> ::= <name> '#'
// <term> ::= <name> '.'
// <meta> ::= <name> ':'
// <macro> ::= <name> '!'
// <method> ::= <name> '(' (<method-disambiguator>)? ').'
// <type-parameter> ::= '[' <name> ']'
// <parameter> ::= '(' <name> ')'
// <name> ::= <identifier>
// <method-disambiguator> ::= <simple-identifier>
// <identifier> ::= <simple-identifier> | <escaped-identifier>
// <simple-identifier> ::= (<identifier-character>)+
// <identifier-character> ::= '_' | '+' | '-' | '$' | ASCII letter or digit
// <escaped-identifier> ::= '`' (<escaped-character>)+ '`', must contain at least one non-<identifier-character>
// <escaped-characters> ::= any UTF-8, escape backticks with double backtick.
// <local-id> ::= <simple-identifier>
// ```
//
// The list of descriptors for a symbol should together form a fully
// qualified name for the symbol. That is, it should serve as a unique
// identifier across the package. Typically, it will include one descriptor
// for every node in the AST (along the ancestry path) between the root of
// the file and the node corresponding to the symbol.
//
// Local symbols MUST only be used for entities which are local to a Document,
// and cannot be accessed from outside the Document.
message Symbol {
string scheme = 1;
Package package = 2;
repeated ScipDescriptor descriptors = 3;
}
// Unit of packaging and distribution.
//
// NOTE: This corresponds to a module in Go and JVM languages.
message Package {
string manager = 1;
string name = 2;
string version = 3;
}
message ScipDescriptor {
enum Suffix {
option allow_alias = true;
UnspecifiedSuffix = 0;
// Unit of code abstraction and/or namespacing.
//
// NOTE: This corresponds to a package in Go and JVM languages.
Namespace = 1;
// Use Namespace instead.
Package = 1 [deprecated = true];
Type = 2;
Term = 3;
Method = 4;
TypeParameter = 5;
Parameter = 6;
// Can be used for any purpose.
Meta = 7;
Local = 8;
Macro = 9;
}
string name = 1;
string disambiguator = 2;
Suffix suffix = 3;
// NOTE: If you add new fields here, make sure to update the prepareSlot()
// function responsible for parsing symbols.
}
// Signature represents the signature of a symbol as it's displayed in API
// documentation or hover tooltips. It uses a subset of Document's fields with
// the same field numbers for wire compatibility with older indexes that encoded
// signatures using the Document message type.
message Signature {
// The language of the signature, e.g. "java", "go", "python".
string language = 4;
// The text content of the signature, e.g. "void add(int a, int b)".
string text = 5;
// (optional) Occurrences within the signature text that reference other
// symbols, enabling hyperlinking of types in the signature. Ranges are
// relative to the `text` field.
repeated Occurrence occurrences = 2;
// Reserved field numbers from the Document message to prevent accidental
// reuse, which would break wire compatibility with older indexes.
reserved 1, 3, 6;
}
// SymbolInformation defines metadata about a symbol, such as the symbol's
// docstring or what package it's defined it.
message SymbolInformation {
// Identifier of this symbol, which can be referenced from `Occurence.symbol`.
// The string must be formatted according to the grammar in `Symbol`.
string symbol = 1;
// (optional, but strongly recommended) The markdown-formatted documentation
// for this symbol. Use `SymbolInformation.signature_documentation` to
// document the method/class/type signature of this symbol.
// Due to historical reasons, indexers may include signature documentation in
// this field by rendering markdown code blocks. New indexers should only
// include non-code documentation in this field, for example docstrings.
repeated string documentation = 3;
// (optional) Relationships to other symbols (e.g., implements, type definition).
repeated Relationship relationships = 4;
// The kind of this symbol. Use this field instead of
// `SymbolDescriptor.Suffix` to determine whether something is, for example, a
// class or a method.
Kind kind = 5;
// (optional) Kind represents the fine-grained category of a symbol, suitable for presenting
// information about the symbol's meaning in the language.
//
// For example:
// - A Java method would have the kind `Method` while a Go function would
// have the kind `Function`, even if the symbols for these use the same
// syntax for the descriptor `SymbolDescriptor.Suffix.Method`.
// - A Go struct has the symbol kind `Struct` while a Java class has
// the symbol kind `Class` even if they both have the same descriptor:
// `SymbolDescriptor.Suffix.Type`.
//
// Since Kind is more fine-grained than Suffix:
// - If two symbols have the same Kind, they should share the same Suffix.
// - If two symbols have different Suffixes, they should have different Kinds.
enum Kind {
UnspecifiedKind = 0;
// A method which may or may not have a body. For Java, Kotlin etc.
AbstractMethod = 66;
// For Ruby's attr_accessor
Accessor = 72;
Array = 1;
// For Alloy
Assertion = 2;
AssociatedType = 3;
// For C++
Attribute = 4;
// For Lean
Axiom = 5;
Boolean = 6;
Class = 7;
// For C++
Concept = 86;
Constant = 8;
Constructor = 9;
// For Solidity
Contract = 62;
// For Haskell
DataFamily = 10;
// For C# and F#
Delegate = 73;
Enum = 11;
EnumMember = 12;
Error = 63;
Event = 13;
// For Dart
Extension = 84;
// For Alloy
Fact = 14;
Field = 15;
File = 16;
Function = 17;
// For 'get' in Swift, 'attr_reader' in Ruby
Getter = 18;
// For Raku
Grammar = 19;
// For Purescript and Lean
Instance = 20;
Interface = 21;
Key = 22;
// For Racket
Lang = 23;
// For Lean
Lemma = 24;
// For solidity
Library = 64;
Macro = 25;
Method = 26;
// For Ruby
MethodAlias = 74;
// Analogous to 'ThisParameter' and 'SelfParameter', but for languages
// like Go where the receiver doesn't have a conventional name.
MethodReceiver = 27;
// Analogous to 'AbstractMethod', for Go.
MethodSpecification = 67;
// For Protobuf
Message = 28;
// For Dart
Mixin = 85;
// For Solidity
Modifier = 65;
Module = 29;
Namespace = 30;
Null = 31;
Number = 32;
Object = 33;
Operator = 34;
Package = 35;
PackageObject = 36;
Parameter = 37;
ParameterLabel = 38;
// For Haskell's PatternSynonyms
Pattern = 39;
// For Alloy
Predicate = 40;
Property = 41;
// Analogous to 'Trait' and 'TypeClass', for Swift and Objective-C
Protocol = 42;
// Analogous to 'AbstractMethod', for Swift and Objective-C.
ProtocolMethod = 68;
// Analogous to 'AbstractMethod', for C++.
PureVirtualMethod = 69;
// For Haskell
Quasiquoter = 43;
// 'self' in Python, Rust, Swift etc.
SelfParameter = 44;
// For 'set' in Swift, 'attr_writer' in Ruby
Setter = 45;
// For Alloy, analogous to 'Struct'.
Signature = 46;
// For Ruby
SingletonClass = 75;
// Analogous to 'StaticMethod', for Ruby.
SingletonMethod = 76;
// Analogous to 'StaticField', for C++
StaticDataMember = 77;
// For C#
StaticEvent = 78;
// For C#
StaticField = 79;
// For Java, C#, C++ etc.
StaticMethod = 80;
// For C#, TypeScript etc.
StaticProperty = 81;
// For C, C++
StaticVariable = 82;
String = 48;
Struct = 49;
// For Swift
Subscript = 47;
// For Lean
Tactic = 50;
// For Lean
Theorem = 51;
// Method receiver for languages
// 'this' in JavaScript, C++, Java etc.
ThisParameter = 52;
// Analogous to 'Protocol' and 'TypeClass', for Rust, Scala etc.
Trait = 53;
// Analogous to 'AbstractMethod', for Rust, Scala etc.
TraitMethod = 70;
// Data type definition for languages like OCaml which use `type`
// rather than separate keywords like `struct` and `enum`.
Type = 54;
TypeAlias = 55;
// Analogous to 'Trait' and 'Protocol', for Haskell, Purescript etc.
TypeClass = 56;
// Analogous to 'AbstractMethod', for Haskell, Purescript etc.
TypeClassMethod = 71;
// For Haskell
TypeFamily = 57;
TypeParameter = 58;
// For C, C++, Capn Proto
Union = 59;
Value = 60;
Variable = 61;
// Next = 87;
// Feel free to open a PR proposing new language-specific kinds.
}
// (optional) The name of this symbol as it should be displayed to the user.
// For example, the symbol "com/example/MyClass#myMethod(+1)." should have the
// display name "myMethod". The `symbol` field is not a reliable source of
// the display name for several reasons:
//
// - Local symbols don't encode the name.
// - Some languages have case-insensitive names, so the symbol is all-lowercase.
// - The symbol may encode names with special characters that should not be
// displayed to the user.
string display_name = 6;
// (optional) The signature of this symbol as it's displayed in API
// documentation or in hover tooltips. For example, a Java method that adds
// two numbers would have `Signature.language = "java"` and
// `Signature.text = "void add(int a, int b)"`. The `language` and `text`
// fields are required while `occurrences` can be optionally included to
// support hyperlinking referenced symbols in the signature.
Signature signature_documentation = 7;
// (optional) The enclosing symbol if this is a local symbol. For non-local
// symbols, the enclosing symbol should be parsed from the `symbol` field
// using the `Descriptor` grammar.
//
// The primary use-case for this field is to allow local symbol to be displayed
// in a symbol hierarchy for API documentation. It's OK to leave this field
// empty for local variables since local variables usually don't belong in API
// documentation. However, in the situation that you wish to include a local
// symbol in the hierarchy, then you can use `enclosing_symbol` to locate the
// "parent" or "owner" of this local symbol. For example, a Java indexer may
// choose to use local symbols for private class fields while providing an
// `enclosing_symbol` to reference the enclosing class to allow the field to
// be part of the class documentation hierarchy. From the perspective of an
// author of an indexer, the decision to use a local symbol or global symbol
// should exclusively be determined whether the local symbol is accessible
// outside the document, not by the capability to find the enclosing
// symbol.
string enclosing_symbol = 8;
}
message Relationship {
string symbol = 1;
// When resolving "Find references", this field documents what other symbols
// should be included together with this symbol. For example, consider the
// following TypeScript code that defines two symbols `Animal#sound()` and
// `Dog#sound()`:
// ```ts
// interface Animal {
// ^^^^^^ definition Animal#
// sound(): string
// ^^^^^ definition Animal#sound()
// }
// class Dog implements Animal {
// ^^^ definition Dog#, relationships = [{symbol: "Animal#", is_implementation: true}]
// public sound(): string { return "woof" }
// ^^^^^ definition Dog#sound(), references_symbols = Animal#sound(), relationships = [{symbol: "Animal#sound()", is_implementation:true, is_reference: true}]
// }
// const animal: Animal = new Dog()
// ^^^^^^ reference Animal#
// console.log(animal.sound())
// ^^^^^ reference Animal#sound()
// ```
// Doing "Find references" on the symbol `Animal#sound()` should return
// references to the `Dog#sound()` method as well. Vice-versa, doing "Find
// references" on the `Dog#sound()` method should include references to the
// `Animal#sound()` method as well.
bool is_reference = 2;
// Similar to `is_reference` but for "Find implementations".
// It's common for `is_implementation` and `is_reference` to both be true but
// it's not always the case.
// In the TypeScript example above, observe that `Dog#` has an
// `is_implementation` relationship with `"Animal#"` but not `is_reference`.
// This is because "Find references" on the "Animal#" symbol should not return
// "Dog#". We only want "Dog#" to return as a result for "Find
// implementations" on the "Animal#" symbol.
bool is_implementation = 3;
// Similar to `references_symbols` but for "Go to type definition".
bool is_type_definition = 4;
// Allows overriding the behavior of "Go to definition" and "Find references"
// for symbols which do not have a definition of their own or could
// potentially have multiple definitions.
//
// For example, in a language with single inheritance and no field overriding,
// inherited fields can reuse the same symbol as the ancestor which declares
// the field. In such a situation, is_definition is not needed.
//
// On the other hand, in languages with single inheritance and some form
// of mixins, you can use is_definition to relate the symbol to the
// matching symbol in ancestor classes, and is_reference to relate the
// symbol to the matching symbol in mixins.
bool is_definition = 5;
// Update registerInverseRelationships on adding a new field here.
}
// SymbolRole declares what "role" a symbol has in an occurrence. A role is
// encoded as a bitset where each bit represents a different role. For example,
// to determine if the `Import` role is set, test whether the second bit of the
// enum value is defined. In pseudocode, this can be implemented with the
// logic: `const isImportRole = (role.value & SymbolRole.Import.value) > 0`.
enum SymbolRole {
// This case is not meant to be used; it only exists to avoid an error
// from the Protobuf code generator.
UnspecifiedSymbolRole = 0;
// Is the symbol defined here? If not, then this is a symbol reference.
Definition = 0x1;
// Is the symbol imported here?
Import = 0x2;
// Is the symbol written here?
WriteAccess = 0x4;
// Is the symbol read here?
ReadAccess = 0x8;
// Is the symbol in generated code?
Generated = 0x10;
// Is the symbol in test code?
Test = 0x20;
// Is this a signature for a symbol that is defined elsewhere?
//
// Applies to forward declarations for languages like C, C++
// and Objective-C, as well as `val` declarations in interface
// files in languages like SML and OCaml.
ForwardDefinition = 0x40;
}
enum SyntaxKind {
option allow_alias = true;
UnspecifiedSyntaxKind = 0;
// Comment, including comment markers and text
Comment = 1;
// `;` `.` `,`
PunctuationDelimiter = 2;
// (), {}, [] when used syntactically
PunctuationBracket = 3;
// `if`, `else`, `return`, `class`, etc.
Keyword = 4;
IdentifierKeyword = 4 [deprecated = true];
// `+`, `*`, etc.
IdentifierOperator = 5;
// non-specific catch-all for any identifier not better described elsewhere
Identifier = 6;
// Identifiers builtin to the language: `min`, `print` in Python.
IdentifierBuiltin = 7;
// Identifiers representing `null`-like values: `None` in Python, `nil` in Go.
IdentifierNull = 8;
// `xyz` in `const xyz = "hello"`
IdentifierConstant = 9;
// `var X = "hello"` in Go
IdentifierMutableGlobal = 10;
// Parameter definition and references
IdentifierParameter = 11;
// Identifiers for variable definitions and references within a local scope
IdentifierLocal = 12;
// Identifiers that shadow other identifiers in an outer scope
IdentifierShadowed = 13;
// Identifier representing a unit of code abstraction and/or namespacing.
//
// NOTE: This corresponds to a package in Go and JVM languages,
// and a module in languages like Python and JavaScript.
IdentifierNamespace = 14;
IdentifierModule = 14 [deprecated = true];
// Function references, including calls
IdentifierFunction = 15;
// Function definition only
IdentifierFunctionDefinition = 16;
// Macro references, including invocations
IdentifierMacro = 17;
// Macro definition only
IdentifierMacroDefinition = 18;
// non-builtin types
IdentifierType = 19;
// builtin types only, such as `str` for Python or `int` in Go
IdentifierBuiltinType = 20;
// Python decorators, c-like __attribute__
IdentifierAttribute = 21;
// `\b`
RegexEscape = 22;
// `*`, `+`
RegexRepeated = 23;
// `.`
RegexWildcard = 24;
// `(`, `)`, `[`, `]`
RegexDelimiter = 25;
// `|`, `-`
RegexJoin = 26;
// Literal strings: "Hello, world!"
StringLiteral = 27;
// non-regex escapes: "\t", "\n"
StringLiteralEscape = 28;
// datetimes within strings, special words within a string, `{}` in format strings
StringLiteralSpecial = 29;
// "key" in { "key": "value" }, useful for example in JSON
StringLiteralKey = 30;
// 'c' or similar, in languages that differentiate strings and characters
CharacterLiteral = 31;
// Literal numbers, both floats and integers
NumericLiteral = 32;
// `true`, `false`
BooleanLiteral = 33;
// Used for XML-like tags
Tag = 34;
// Attribute name in XML-like tags
TagAttribute = 35;
// Delimiters for XML-like tags
TagDelimiter = 36;
}
// SingleLineRange represents a half-open [start, end) range within a single line.
//
// Line numbers and characters are always 0-based. Make sure to increment them
// before displaying in an editor-like UI because editors conventionally use
// 1-based numbers. The `character` values are interpreted based on the
// `PositionEncoding` for the enclosing Document.
message SingleLineRange {
int32 line = 1;
int32 start_character = 2;
int32 end_character = 3;
}
// MultiLineRange represents a half-open [start, end) range spanning multiple lines.
//
// Line numbers and characters are always 0-based. Make sure to increment them
// before displaying in an editor-like UI because editors conventionally use
// 1-based numbers. The `character` values are interpreted based on the
// `PositionEncoding` for the enclosing Document.
//
// Producers SHOULD use `SingleLineRange` when `start_line == end_line` to keep
// indexes compact, but consumers MUST accept multi-line encoding even when the
// range happens to fit on a single line.
message MultiLineRange {
int32 start_line = 1;
int32 start_character = 2;
int32 end_line = 3;
int32 end_character = 4;
}
// Occurrence associates a source position with a symbol and/or highlighting
// information.
//
// If possible, indexers should try to bundle logically related information
// across occurrences into a single occurrence to reduce payload sizes.
//
// Range encoding:
//
// An Occurrence carries its source range in one of two ways: the deprecated
// `range` field (a `repeated int32` packed encoding kept for backward
// compatibility), or one of the typed alternatives in the `typed_range`
// oneof. New producers SHOULD set `typed_range` and SHOULD NOT set the
// deprecated `range` field. The same rule applies to `enclosing_range` and
// `typed_enclosing_range`.
//
// When both encodings are present on the same Occurrence, `typed_range` takes
// precedence over `range` (likewise `typed_enclosing_range` over
// `enclosing_range`). Producers that set both forms MUST keep them
// semantically equivalent. Consumers SHOULD prefer the typed form when
// available and fall back to the `repeated int32` form otherwise.
message Occurrence {
// Deprecated: Use `single_line_range` or `multi_line_range` instead.
//
// Half-open [start, end) range. Must be exactly three or four elements:
// - Three elements: `[startLine, startCharacter, endCharacter]` (single-line)
// - Four elements: `[startLine, startCharacter, endLine, endCharacter]`
//
// The end line of a three-element range is inferred to equal the start line.
//
// Historical note: the original draft of this schema had a `Range` message
// type with `start` and `end` fields of type `Position`, mirroring LSP.
// Benchmarks revealed that this encoding was inefficient and that we could
// reduce the total payload size of an index by 50% by using `repeated int32`
// instead. However, the lack of type safety led to the introduction of
// `single_line_range` and `multi_line_range` as typed alternatives; the
// typed encoding's per-index size overhead is small (single-digit percent)
// because ranges are only a fraction of a typical index payload.
repeated int32 range = 1 [deprecated = true];
// Half-open [start, end) source range of this occurrence.
//
// It is allowed for the range to be empty (i.e. start==end).
//
// When both `typed_range` and the deprecated `range` field are set,
// `typed_range` takes precedence.
oneof typed_range {
// Range spanning a single line.
SingleLineRange single_line_range = 8;
// Range spanning multiple lines.
MultiLineRange multi_line_range = 9;
}
// (optional) The symbol that appears at this position. See
// `SymbolInformation.symbol` for how to format symbols as strings.
string symbol = 2;
// (optional) Bitset containing `SymbolRole`s in this occurrence.
// See `SymbolRole`'s documentation for how to read and write this field.
int32 symbol_roles = 3;
// (optional) CommonMark-formatted documentation for this specific range. If
// empty, the `Symbol.documentation` field is used instead. One example
// where this field might be useful is when the symbol represents a generic
// function (with abstract type parameters such as `List<T>`) and at this
// occurrence we know the exact values (such as `List<String>`).
//
// This field can also be used for dynamically or gradually typed languages,
// which commonly allow for type-changing assignment.
repeated string override_documentation = 4;
// (optional) What syntax highlighting class should be used for this range?
SyntaxKind syntax_kind = 5;
// (optional) Diagnostics that have been reported for this specific range.
repeated Diagnostic diagnostics = 6;
// Deprecated: Use `typed_enclosing_range` instead.
//
// Uses the same `repeated int32` encoding as the deprecated `range` field.
repeated int32 enclosing_range = 7 [deprecated = true];
// (optional) Half-open source range of the nearest non-trivial enclosing AST
// node. This range must enclose the occurrence range. Example applications:
//
// - Call hierarchies: to determine what symbols are referenced from the body
// of a function
// - Symbol outline: to display breadcrumbs from the cursor position to the
// root of the file
// - Expand selection: to select the nearest enclosing AST node.
// - Highlight range: to indicate the AST expression that is associated with a
// hover popover
//
// For definition occurrences, the enclosing range should indicate the
// start/end bounds of the entire definition AST node, including
// documentation.
// ```
// const n = 3
// ^ range
// ^^^^^^^^^^^ enclosing_range
//
// /** Parses the string into something */
// ^ enclosing_range start --------------------------------------|
// function parse(input string): string { |
// ^^^^^ range |
// return input.slice(n) |
// } |
// ^ enclosing_range end <---------------------------------------|
// ```
//
// Any attributes/decorators/attached macros should also be part of the
// enclosing range.
//
// ```python
// @cache
// ^ enclosing_range start---------------------|
// def factorial(n): |
// return n * factorial(n-1) if n else 1 |
// < enclosing_range end-----------------------|
//
// ```
//
// For reference occurrences, the enclosing range should indicate the start/end
// bounds of the parent expression.
// ```
// const a = a.b
// ^ range
// ^^^ enclosing_range
// const b = a.b(41).f(42).g(43)
// ^ range
// ^^^^^^^^^^^^^ enclosing_range
// ```
//
// When both `typed_enclosing_range` and the deprecated `enclosing_range`
// field are set, `typed_enclosing_range` takes precedence.
oneof typed_enclosing_range {
// Enclosing range spanning a single line.
SingleLineRange single_line_enclosing_range = 10;
// Enclosing range spanning multiple lines.
MultiLineRange multi_line_enclosing_range = 11;
}
}
// Represents a diagnostic, such as a compiler error or warning, which should be
// reported for a document.
message Diagnostic {
// Should this diagnostic be reported as an error, warning, info, or hint?
Severity severity = 1;
// (optional) Code of this diagnostic, which might appear in the user interface.
string code = 2;
// Message of this diagnostic.
string message = 3;
// (optional) Human-readable string describing the source of this diagnostic, e.g.
// 'typescript' or 'super lint'.
string source = 4;
repeated DiagnosticTag tags = 5;
}
enum Severity {
UnspecifiedSeverity = 0;
Error = 1;
Warning = 2;
Information = 3;
Hint = 4;
}
enum DiagnosticTag {
UnspecifiedDiagnosticTag = 0;
Unnecessary = 1;
Deprecated = 2;
}
// Language standardises names of common programming languages that can be used
// for the `Document.language` field. The primary purpose of this enum is to
// prevent a situation where we have a single programming language ends up with
// multiple string representations. For example, the C++ language uses the name
// "CPP" in this enum and other names such as "cpp" are incompatible.
// Feel free to send a pull-request to add missing programming languages.
enum Language {
UnspecifiedLanguage = 0;
ABAP = 60;
Apex = 96;
APL = 49;
Ada = 39;
Agda = 45;
AsciiDoc = 86;
Assembly = 58;
Awk = 66;
Bat = 68;
BibTeX = 81;
C = 34;
COBOL = 59;
CPP = 35; // C++ (the name "CPP" was chosen for consistency with LSP)
CSS = 26;
CSharp = 1;
Clojure = 8;
Coffeescript = 21;
CommonLisp = 9;
Coq = 47;
CUDA = 97;
Dart = 3;
Delphi = 57;
Diff = 88;
Dockerfile = 80;
Dyalog = 50;
Elixir = 17;
Erlang = 18;
FSharp = 42;
Fish = 65;
Flow = 24;
Fortran = 56;
Git_Commit = 91;
Git_Config = 89;
Git_Rebase = 92;
Go = 33;
GraphQL = 98;
Groovy = 7;
HTML = 30;
Hack = 20;
Handlebars = 90;
Haskell = 44;
Idris = 46;
Ini = 72;
J = 51;
JSON = 75;
Java = 6;
JavaScript = 22;
JavaScriptReact = 93;
Jsonnet = 76;
Julia = 55;
Justfile = 109;
Kotlin = 4;
LaTeX = 83;
Lean = 48;
Less = 27;
Lua = 12;
Luau = 108;
Makefile = 79;
Markdown = 84;
Matlab = 52;
Nickel = 110; // https://nickel-lang.org/
Nix = 77;
OCaml = 41;
Objective_C = 36;
Objective_CPP = 37;
Odin = 111; // https://odin-lang.org/
Pascal = 99;
PHP = 19;
PLSQL = 70;
Perl = 13;
PowerShell = 67;
Prolog = 71;
Protobuf = 100;
Python = 15;
R = 54;
Racket = 11;
Raku = 14;
Razor = 62;
Repro = 102; // Internal language for testing SCIP
ReST = 85;
Ruby = 16;
Rust = 40;
SAS = 61;
SCSS = 29;
SML = 43;
SQL = 69;
Sass = 28;
Scala = 5;
Scheme = 10;
ShellScript = 64; // Bash
Skylark = 78;
Slang = 107;
Solidity = 95;
Svelte = 106;
Swift = 2;
Tcl = 101;
TOML = 73;
TeX = 82;
Thrift = 103;
TypeScript = 23;
TypeScriptReact = 94;
Verilog = 104;
VHDL = 105;
VisualBasic = 63;
Vue = 25;
Wolfram = 53;
XML = 31;
XSL = 32;
YAML = 74;
Zig = 38;
// NextLanguage = 112;
// Steps add a new language:
// 1. Copy-paste the "NextLanguage = N" line above
// 2. Increment "NextLanguage = N" to "NextLanguage = N+1"
// 3. Replace "NextLanguage = N" with the name of the new language.
// 4. Move the new language to the correct line above using alphabetical order
// 5. (optional) Add a brief comment behind the language if the name is not self-explanatory
}

6
PythonProject/.idea/vcs.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$/.." vcs="Git" />
</component>
</project>

13
PythonProject/Dockerfile Normal file
View File

@@ -0,0 +1,13 @@
FROM python:3.10-slim
WORKDIR /app
# Установка зависимостей
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Копирование кода приложения
COPY . .
# Команда запуска (замени main:app или script.py на свой стартовый файл)
CMD ["python", "main.py"]

3
PythonProject/index.scip Normal file
View File

@@ -0,0 +1,3 @@
+
scip-python0.6.6file:///workspace 

0
PythonProject/pyproject.toml Normal file
View File

23
README.md
View File

@@ -1,5 +1,24 @@
# CodeBase # CodeBase
Прототип базы знаний по коду проектов. Парсит проект на чанки согласно синтаксическому дереву, векторизует через GrafCodeBert, потом может ответить на вопрос по коду, сравнивая вектора вопроса и чанков, ответ формируется с помощью LLM.
Прототип базы знаний по коду проектов. На вход получает полный адрес .cproj файла, парсит код для составления графа с векторным представлением каждого метода. Способен ответить на вопрос по проекту, ища векторное совпадение по косинусному сходству и по взаимосвязям в графе
Patch Notes: Patch Notes:
v1: реализовано MVP, работаем только с шарпом, храним все в оперативе. В качестве LLM - qwen2.5-coder:7
* v1: реализовано MVP, работаем только с шарпом, храним все в оперативе. В качестве LLM - qwen2.5-coder:7
* v2:
* Переезд на графовую базу данных Neo4j с автоматической инициализацией.
* С помощью Microsoft.CodeAnalysis.MSBuild парсит все зависимости, исключая попадание системных вызовов в дерево вызовов кода.
* Изменен промт: теперь анализируем не просто чанки, а дерево зависимостей
* Вся инфраструктура теперь в докере: база, питоновский скрипт для векторизации, llm
* НУЖЕН РЕФАКТОРИНГ
* НУЖЕН ПЕРЕСМОТР СКАЧИВАНИЯ LLM Внутри Ollama - долго
* Нужна проверка бага: парсинг сохраняется не сразу
* v2.1.1:
* Реализована попытка в мультипарсинг: шарп, с горем пополам питон, должен еще го и ts
* почищены лишние файлы, новая репа для работы с бд, отдельно вынесена иинициализация
* теперь ручка парсера принимает на вход путь к проекту, название и язык в виде enum
* расширение списка языков проиходит через enum и настройку appsetting посредством добавления строки вида язык:команда докера для сборки файла index через Scip
* установлен protobuf для работы с Scip

46
docker-compose.yml Normal file
View File

@@ -0,0 +1,46 @@
version: '3.8'
services:
neo4j:
image: neo4j:5.15.0
container_name: graphrag_neo4j
ports:
- "7474:7474"
- "7687:7687"
environment:
- NEO4J_AUTH=neo4j/password123
python-service:
build: ./PythonProject
container_name: graphrag_python
ports:
- "8000:8000"
ollama:
image: ollama/ollama:latest
container_name: graphrag_ollama
ports:
- "11434:11434"
volumes:
- ollama_data:/root/.ollama
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]
ollama-init:
image: ollama/ollama:latest
container_name: graphrag_ollama_init
depends_on:
- ollama # Ждем, пока запустится основной сервер
environment:
# Указываем этому контейнеру стучаться в основной сервер
- OLLAMA_HOST=ollama:11434
# Команда на скачивание (замени qwen2.5-coder на нужную, если передумаешь)
command: pull qwen2.5-coder:7b
volumes:
ollama_data: