Skip to content

Commit

Permalink
Redo implementation of JSON compilation database
Browse files Browse the repository at this point in the history
  • Loading branch information
@GalaxyShard
GalaxyShard committed Nov 18, 2024
1 parent 30682d6 commit 48930ce
Show file tree
Hide file tree
Showing 3 changed files with 152 additions and 123 deletions.
4 changes: 3 additions & 1 deletion lib/compiler/build_runner.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,6 @@ pub fn main() !void {
graph.allow_so_scripts = false;
} else if (mem.eql(u8, arg, "-fcompdb")) {
builder.enable_compdb = true;
try builder.initCompdb();
} else if (mem.eql(u8, arg, "-fno-compdb")) {
builder.enable_compdb = false;
} else if (mem.eql(u8, arg, "-freference-trace")) {
Expand Down Expand Up @@ -322,6 +321,9 @@ pub fn main() !void {
try targets.append(arg);
}
}
if (builder.enable_compdb) {
try builder.initCompdb();
}

const stderr = std.io.getStdErr();
const ttyconf = get_tty_conf(color, stderr);
Expand Down
149 changes: 111 additions & 38 deletions lib/std/Build.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,7 @@ pkg_hash: []const u8,
/// A mapping from dependency names to package hashes.
available_deps: AvailableDeps,

compdb_entries_dir: ?[]const u8 = null,

compdb_entry_paths_set: ?*StringHashMap(void) = null,
compile_commands_database: ?*CompileCommandsDatabase = null,

release_mode: ReleaseMode,

Expand All @@ -111,6 +109,16 @@ pub const ReleaseMode = enum {
small,
};

pub const CompileCommandsDatabase = struct {
entries: ArrayList(CompileCommandsEntry),
};
pub const CompileCommandsEntry = struct {
module: *Module,
working_directory: []const u8,
relative_path: []const u8,
flags: []const []const u8,
};

/// Shared state among all Build instances.
/// Settings that are here rather than in Build are not configurable per-package.
pub const Graph = struct {
Expand Down Expand Up @@ -407,8 +415,7 @@ fn createChildOnly(
.named_lazy_paths = .init(allocator),
.pkg_hash = pkg_hash,
.available_deps = pkg_deps,
.compdb_entries_dir = parent.compdb_entries_dir,
.compdb_entry_paths_set = parent.compdb_entry_paths_set,
.compile_commands_database = parent.compile_commands_database,
.release_mode = parent.release_mode,
};
try child.top_level_steps.put(allocator, child.install_tls.step.name, &child.install_tls);
Expand Down Expand Up @@ -658,50 +665,116 @@ fn determineAndApplyInstallPrefix(b: *Build) !void {
pub fn initCompdb(self: *Build) !void {
if (!self.enable_compdb) return;

self.compdb_entries_dir = self.makeTempPath();

const entry_paths_set = try self.allocator.create(StringHashMap(void));
entry_paths_set.* = StringHashMap(void).init(self.allocator);
self.compdb_entry_paths_set = entry_paths_set;
const database = try self.allocator.create(CompileCommandsDatabase);
database.* = .{
.entries = ArrayList(CompileCommandsEntry).init(self.allocator),
};
self.compile_commands_database = database;
}

pub fn generateCompdb(self: *std.Build) !void {
if (!self.enable_compdb) return;

var entry_strings = ArrayList(u8).init(self.allocator);
defer entry_strings.deinit();
var file = try self.build_root.handle.createFile("compile_commands.json", .{});
defer file.close();
var buffered = std.io.bufferedWriter(file.writer());
var writer = buffered.writer();

var entry_paths_set_key_it = self.compdb_entry_paths_set.?.keyIterator();
while (entry_paths_set_key_it.next()) |entry_path| {
// 8MiB max size should be more than enough for a single entry in 99.99999+% of cases.
// TODO: Maybe this shouldn't silently skip entries that are too large?
const entry_file_contents_0: []u8 = fs.cwd().readFileAlloc(self.allocator, entry_path.*, (1024 * 1024 * 8)) catch continue;
defer self.allocator.free(entry_file_contents_0);
var entry_file_contents_1: []const u8 = entry_file_contents_0;
entry_file_contents_1 = mem.trim(u8, entry_file_contents_1, " \n\r,");
if (entry_file_contents_1.len == 0) continue;
entry_file_contents_1 = self.fmt("{s},", .{entry_file_contents_1});
_ = try writer.write("[");

entry_strings.appendSlice(entry_file_contents_1) catch continue;
}
const entries = self.compile_commands_database.?.entries.items;

var entries_string: []const u8 = entry_strings.items;
if (entries_string.len != 0) {
entries_string = mem.trimRight(u8, entries_string, ",");
}
entries_string = self.fmt("[{s}]", .{entries_string});
var temp = std.ArrayList(u8).init(self.allocator);
defer temp.deinit();

try self.build_root.handle.writeFile(.{
.data = entries_string,
.sub_path = "compile_commands.json",
});
}
for (entries, 0..) |entry, i| {
_ = try writer.write("{");

_ = try writer.write("\"directory\":");
try std.json.encodeJsonString(entry.working_directory, .{}, writer);
_ = try writer.write(",");

_ = try writer.write("\"arguments\":[");
_ = try writer.write("\"zig\",");
_ = try writer.write("\"cc\",");

pub fn destroy(b: *Build) void {
if (b.compdb_entry_paths_set) |set| {
set.deinit();
b.allocator.destroy(set);
for (entry.flags) |flag| {
try std.json.encodeJsonString(flag, .{}, writer);
_ = try writer.write(",");
}
for (entry.module.c_macros.items) |macro_flag| {
try std.json.encodeJsonString(macro_flag, .{}, writer);
_ = try writer.write(",");
}
for (entry.module.include_dirs.items) |include_dir| {
const args: [2][]const u8 = blk: switch (include_dir) {
.path => |include_path| {
break :blk .{ "-I", include_path.getPath2(self, null) };
},
.path_system => |include_path| {
break :blk .{ "-isystem", include_path.getPath2(self, null) };
},
.path_after => |include_path| {
break :blk .{ "-idirafter", include_path.getPath2(self, null) };
},
.framework_path => |include_path| {
break :blk .{ "-F", include_path.getPath2(self, null) };
},
.framework_path_system => |include_path| {
break :blk .{ "-iframework", include_path.getPath2(self, null) };
},
.other_step => |other| {
if (other.generated_h) |header| {
break :blk .{ "-isystem", std.fs.path.dirname(header.getPath()).? };
}
if (other.installed_headers_include_tree) |include_tree| {
break :blk .{ "-I", include_tree.generated_directory.getPath() };
}
continue;
},
.config_header_step => |config_header| {
const full_file_path = config_header.output_file.getPath();
const header_dir_path = full_file_path[0 .. full_file_path.len - config_header.include_path.len];
break :blk .{ "-I", header_dir_path };
},
};
try std.json.encodeJsonString(args[0], .{}, writer);
_ = try writer.write(",");
try std.json.encodeJsonString(args[1], .{}, writer);
_ = try writer.write(",");
}

if (entry.module.resolved_target) |*target| {
if (!target.query.isNative()) {
const triple = try target.query.zigTriple(self.allocator);
const cpu = try target.query.serializeCpuAlloc(self.allocator);

try temp.appendSlice("--target=");
try temp.appendSlice(triple);
try std.json.encodeJsonString(temp.items, .{}, writer);
temp.clearRetainingCapacity();

try temp.appendSlice("-mcpu=");
try temp.appendSlice(cpu);
try std.json.encodeJsonString(temp.items, .{}, writer);
}
}

try std.json.encodeJsonString(entry.relative_path, .{}, writer);
_ = try writer.write("],");

_ = try writer.write("\"file\":");
try std.json.encodeJsonString(entry.relative_path, .{}, writer);

_ = try writer.write("}");
if (i != entries.len-1) {
_ = try writer.write(",");
}
}

_ = try writer.write("]");

try buffered.flush();
}

/// This function is intended to be called by lib/build_runner.zig, not a build.zig file.
Expand Down
122 changes: 38 additions & 84 deletions lib/std/Build/Step/Compile.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ const assert = std.debug.assert;
const panic = std.debug.panic;
const ArrayList = std.ArrayList;
const StringHashMap = std.StringHashMap;
const CityHash64 = std.hash.CityHash64;
const Sha256 = std.crypto.hash.sha2.Sha256;
const Allocator = mem.Allocator;
const Step = std.Build.Step;
Expand Down Expand Up @@ -1265,97 +1264,67 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
.c_source_file => |c_source_file| l: {
if (!my_responsibility) break :l;

const c_source_file_path = c_source_file.file.getPath2(dep.module.owner, step);
const c_source_file_compdb_entry_path: ?[]u8 = if (b.enable_compdb)
generateCSourceFileCompdbEntryPath(b, c_source_file_path)
else
null;
if (c_source_file_compdb_entry_path) |compdb_entry_path| {
try b.compdb_entry_paths_set.?.put(compdb_entry_path, {});
}

if (c_source_file.flags.len == 0) {
if (prev_has_cflags) {
try zig_args.append("-cflags");
if (c_source_file_compdb_entry_path) |compdb_entry_path| {
try zig_args.append(b.fmt("-MJ{s}", .{compdb_entry_path}));
}
try zig_args.append("--");

prev_has_cflags = if (c_source_file_compdb_entry_path) |_|
true
else
false;
} else if (c_source_file_compdb_entry_path) |compdb_entry_path| {
try zig_args.append("-cflags");
try zig_args.append(b.fmt("-MJ{s}", .{compdb_entry_path}));
try zig_args.append("--");

prev_has_cflags = true;
prev_has_cflags = false;
}
} else {
try zig_args.append("-cflags");
if (c_source_file_compdb_entry_path) |compdb_entry_path| {
try zig_args.append(b.fmt("-MJ{s}", .{compdb_entry_path}));
}
for (c_source_file.flags) |arg| {
try zig_args.append(arg);
}
try zig_args.append("--");

prev_has_cflags = true;
}
try zig_args.append(c_source_file_path);
const path = c_source_file.file.getPath3(dep.module.owner, step);
if (b.compile_commands_database) |database| {
try database.entries.append(.{
.module = &compile.root_module,
.working_directory = b.pathResolve(&.{path.root_dir.path orelse "."}),
.relative_path = path.sub_path,
.flags = c_source_file.flags,
});

}
try zig_args.append(c_source_file.file.getPath2(dep.module.owner, step));
total_linker_objects += 1;
},

.c_source_files => |c_source_files| l: {
if (!my_responsibility) break :l;

const root_path = c_source_files.root.getPath2(dep.module.owner, step);
for (c_source_files.files) |file| {
const c_source_file_path = b.pathJoin(&.{ root_path, file });

const c_source_file_compdb_entry_path: ?[]u8 = if (b.enable_compdb)
generateCSourceFileCompdbEntryPath(b, c_source_file_path)
else
null;
if (c_source_file_compdb_entry_path) |compdb_entry_path| {
try b.compdb_entry_paths_set.?.put(compdb_entry_path, {});
}

if (c_source_files.flags.len == 0) {
if (prev_has_cflags) {
try zig_args.append("-cflags");
if (c_source_file_compdb_entry_path) |compdb_entry_path| {
try zig_args.append(b.fmt("-MJ{s}", .{compdb_entry_path}));
}
try zig_args.append("--");

prev_has_cflags = if (c_source_file_compdb_entry_path) |_|
true
else
false;
} else if (c_source_file_compdb_entry_path) |compdb_entry_path| {
try zig_args.append("-cflags");
try zig_args.append(b.fmt("-MJ{s}", .{compdb_entry_path}));
try zig_args.append("--");

prev_has_cflags = true;
}
} else {
if (c_source_files.flags.len == 0) {
if (prev_has_cflags) {
try zig_args.append("-cflags");
if (c_source_file_compdb_entry_path) |compdb_entry_path| {
try zig_args.append(b.fmt("-MJ{s}", .{compdb_entry_path}));
}
for (c_source_files.flags) |arg| {
try zig_args.append(arg);
}
try zig_args.append("--");
prev_has_cflags = false;
}
} else {
try zig_args.append("-cflags");
for (c_source_files.flags) |flag| {
try zig_args.append(flag);
}
try zig_args.append("--");
prev_has_cflags = true;
}

prev_has_cflags = true;
const root_path = c_source_files.root.getPath2(dep.module.owner, step);

if (b.compile_commands_database) |database| {
for (c_source_files.files) |file| {
try database.entries.append(.{
.module = &compile.root_module,
.working_directory = root_path,
.relative_path = file,
.flags = c_source_files.flags,
});
}
try zig_args.append(c_source_file_path);
}

for (c_source_files.files) |file| {
try zig_args.append(b.pathJoin(&.{ root_path, file }));
}

total_linker_objects += c_source_files.files.len;
Expand Down Expand Up @@ -1913,21 +1882,6 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
}
}

fn generateCSourceFileCompdbEntryPath(
b: *std.Build,
c_source_file_path: []const u8,
) []u8 {
const c_source_file_path_basename = fs.path.basename(c_source_file_path);
const c_source_file_path_basename_hash = std.Build.hex64(CityHash64.hash(c_source_file_path_basename));

return b.fmt("{s}{c}{s}-{s}", .{
b.compdb_entries_dir.?,
std.fs.path.sep,
c_source_file_path_basename,
c_source_file_path_basename_hash,
});
}

pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) !Path {
const gpa = c.step.owner.allocator;

Expand Down

0 comments on commit 48930ce

Please sign in to comment.