diff --git a/.vscode/settings.json b/.vscode/settings.json index 52d602c307..9b19c6bc9e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,6 +5,7 @@ "${workspaceFolder}/capture", "${workspaceFolder}/csvexport", "${workspaceFolder}/import", + "${workspaceFolder}/multiprocess", "${workspaceFolder}/update", "${workspaceFolder}/test", "${workspaceFolder}", diff --git a/multiprocess/CMakeLists.txt b/multiprocess/CMakeLists.txt new file mode 100644 index 0000000000..fe0fa91b9c --- /dev/null +++ b/multiprocess/CMakeLists.txt @@ -0,0 +1,25 @@ +cmake_minimum_required(VERSION 3.16) + +option(NO_ISA_EXTENSIONS "Disable ISA extensions (don't pass -march=native or -mcpu=native to the compiler)" OFF) +option(NO_STATISTICS OFF) # we need those to get processed source zone locations + +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/version.cmake) + +set(CMAKE_CXX_STANDARD 20) + +project( + tracy-multiprocess + LANGUAGES C CXX ASM + VERSION ${TRACY_VERSION_STRING} +) + +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/config.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/vendor.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/server.cmake) + + +add_executable(tracy-merge + src/merger.cpp) +target_link_libraries(tracy-merge PRIVATE TracyServer TracyGetOpt) + +set_property(DIRECTORY ${CMAKE_CURRENT_LIST_DIR} PROPERTY VS_STARTUP_PROJECT ${PROJECT_NAME}) diff --git a/multiprocess/src/merger.cpp b/multiprocess/src/merger.cpp new file mode 100644 index 0000000000..d00146b226 --- /dev/null +++ b/multiprocess/src/merger.cpp @@ -0,0 +1,337 @@ +#include "TracyFileRead.hpp" +#include "TracyFileWrite.hpp" +#include "TracyWorker.hpp" +#include "getopt.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std::chrono_literals; + +bool orderImportTimelineEvents( tracy::Worker::ImportEventTimeline const& a, + tracy::Worker::ImportEventTimeline const& b ) +{ + return a.timestamp < b.timestamp; +} + +bool orderImportMessageEvents( tracy::Worker::ImportEventMessages const& a, + tracy::Worker::ImportEventMessages const& b ) +{ + return a.timestamp < b.timestamp; +} + +struct ExportedWorker +{ + std::vector timeline; + std::vector messages; + std::vector plots; + + std::unordered_map threadNames; + std::string name; + std::string process; + + static ExportedWorker merge( std::vector inputList ) + { + uint64_t nextFreeThreadId = 1; + ExportedWorker out; + + // for some data, arbitrarily take the infos from the first trace + auto const& firstExport = *inputList[0]; + out.name = firstExport.name; + out.process = firstExport.process; + + // quick pass to allocate output vectors + size_t numTimelineEvents = 0, numMessages = 0, numPlots = 0; + for( auto const& inputWorker : inputList ) + { + numTimelineEvents += inputWorker->timeline.size(); + numMessages += inputWorker->messages.size(); + numPlots += inputWorker->plots.size(); + } + out.timeline.reserve( numTimelineEvents ); + out.messages.reserve( numMessages ); + out.plots.reserve( numPlots ); + + size_t eventsSortedSoFar = 0; + size_t messagesSortedSoFar = 0; + + // keep track of registered threads to avoid overlaps + std::unordered_map localThreadToMultiprocess; + + for( auto exportPtr : inputList ) + { + ExportedWorker const& exported = *exportPtr; + + // rebuild thread mapping + // we try to keep original thread IDs intact if possible, falling back to made-up IDs + // in case of conflict + for( auto const& threadId : exported.threadNames ) + { + uint64_t multiprocessId = threadId.first; + if( localThreadToMultiprocess.contains( multiprocessId ) ) + { + // oh-oh, conflict - let's take a random ID instead; + multiprocessId = nextFreeThreadId++; + } + localThreadToMultiprocess[threadId.first] = multiprocessId; + out.threadNames[multiprocessId] = threadId.second; + } + + // translate all events with the right thread IDs + for( auto&& event : exported.timeline ) + { + tracy::Worker::ImportEventTimeline& inserted = out.timeline.emplace_back( event ); + inserted.tid = localThreadToMultiprocess[inserted.tid]; + } + for( auto&& message : exported.messages ) + { + tracy::Worker::ImportEventMessages& inserted = out.messages.emplace_back( message ); + inserted.tid = localThreadToMultiprocess[inserted.tid]; + } + for( auto&& plots : exported.plots ) + { + out.plots.emplace_back( plots ); + } + + // sort timeline and messages events + std::inplace_merge( out.timeline.begin(), out.timeline.begin() + eventsSortedSoFar, out.timeline.end(), + orderImportTimelineEvents ); + eventsSortedSoFar += exported.timeline.size(); + std::inplace_merge( out.messages.begin(), out.messages.begin() + messagesSortedSoFar, out.messages.end(), + orderImportMessageEvents ); + messagesSortedSoFar += exported.messages.size(); + } + return out; + } + + static std::optional fromTracyFile( std::string const& filepath, bool exportPlots ) + { + std::unique_ptr sourceFile{ tracy::FileRead::Open( ( filepath.c_str() ) ) }; + if( !sourceFile ) + { + std::cerr << "Could not find file" << std::endl; + return std::nullopt; + } + std::cout << "reading " << filepath << std::endl; + + tracy::Worker worker{ *sourceFile, tracy::EventType::All, + true, // otherwise source zones are empty + false }; + while( !worker.AreSourceLocationZonesReady() ) + { + std::cout << "Waiting for source locations" << std::endl; + std::this_thread::sleep_for( 1s ); + } + + ExportedWorker exportedData; + exportedData.name = worker.GetCaptureName(); + exportedData.process = worker.GetCaptureProgram(); + std::cout << exportedData.name << " (" << exportedData.process << ")" << std::endl; + + std::unordered_set seenThreadIds; + + auto& sourceLocationZones = worker.GetSourceLocationZones(); + std::cout << "- " << sourceLocationZones.size() << " events" << std::endl; + for( auto&& zone_it : sourceLocationZones ) + { + const tracy::SourceLocation& sourceLoc = worker.GetSourceLocation( zone_it.first ); + std::string zoneFilePath = worker.GetString( sourceLoc.file ); + int zoneLine = sourceLoc.line; + std::string zoneName = worker.GetZoneName( sourceLoc ); + + auto const& zones = zone_it.second; + for( auto&& zoneData : zones.zones ) + { + const auto zone_event = zoneData.Zone(); + const uint64_t threadFullId = worker.DecompressThread( zoneData.Thread() ); + const auto start = zone_event->Start(); + const auto end = zone_event->End(); + seenThreadIds.emplace( threadFullId ); + + auto& startEvent = exportedData.timeline.emplace_back(); + startEvent.locFile = zoneFilePath; + startEvent.locLine = zoneLine; + startEvent.name = zoneName; + startEvent.tid = threadFullId; + startEvent.isEnd = false; + startEvent.timestamp = zone_event->Start(); + + auto& endEvent = exportedData.timeline.emplace_back(); + endEvent.locFile = zoneFilePath; + endEvent.locLine = zoneLine; + endEvent.name = zoneName; + endEvent.tid = threadFullId; + endEvent.isEnd = true; + endEvent.timestamp = zone_event->End(); + } + } + // need to sort because we split 'begin' and 'end' events + std::sort( exportedData.timeline.begin(), exportedData.timeline.end(), orderImportTimelineEvents ); + + auto const& messages = worker.GetMessages(); + std::cout << "- " << messages.size() << " messages" << std::endl; + for( auto const& messages_it : worker.GetMessages() ) + { + tracy::MessageData const& messageData = *messages_it; + tracy::Worker::ImportEventMessages importMessage; + uint64_t const threadId = worker.DecompressThread( messageData.thread ); + importMessage.tid = threadId; + importMessage.message = worker.GetString( messageData.ref ); + importMessage.timestamp = messageData.time; + + exportedData.messages.push_back( importMessage ); + seenThreadIds.emplace( threadId ); + } + // to be sure, but should not do a lot + std::sort( exportedData.messages.begin(), exportedData.messages.end(), orderImportMessageEvents ); + + if( exportPlots ) + { + auto const& plots = worker.GetPlots(); + std::cout << "- " << plots.size() << " plots" << std::endl; + for( auto const& plots_it : worker.GetPlots() ) + { + tracy::Worker::ImportEventPlots importPlot; + importPlot.name = worker.GetString( plots_it->name ); + importPlot.format = plots_it->format; + + importPlot.data.resize( plots_it->data.size() ); + for( auto const& elt : plots_it->data ) + { + std::pair dataPoint{ elt.time.Val(), elt.val }; + importPlot.data.push_back( dataPoint ); + } + exportedData.plots.push_back( importPlot ); + } + } + + for( auto&& tid : seenThreadIds ) + { + std::string name = worker.GetThreadName( tid ); + exportedData.threadNames[tid] = exportedData.process + "/" + name; + } + + return exportedData; + } +}; + +[[noreturn]] void Usage() +{ + printf( "Usage: merge [-fp] -o output.tracy input1.tracy [input2.tracy]...\n\n" ); + printf( "Options\n" ); + printf( " --output/-o Output file path\n" ); + printf( " --force/-f Overwrite output file if it exists\n" ); + printf( " --export-plots/-p (experimental) Also exports plots\n" ); + exit( 1 ); +} + +struct Args +{ + std::vector inputPaths; + std::string outputPath; + bool exportPlots = false; + + static Args parse( int argc, char* argv[] ) + { + Args args; + // option parsing + bool overwrite = false; + int c; + const struct option long_options[] = { + { "output", required_argument, 0, 'o' }, + { "force", no_argument, 0, 'f' }, + { "export-plots", no_argument, 0, 'p' }, + { 0, 0, 0, 0 }, + }; + while( ( c = getopt_long( argc, argv, "o:fp", long_options, nullptr ) ) != -1 ) + { + switch( c ) + { + case 'o': + args.outputPath = optarg; + break; + case 'f': + overwrite = true; + break; + case 'p': + args.exportPlots = true; + break; + default: + Usage(); + break; + } + } + for( int argIndex = optind; argIndex < argc; argIndex++ ) + { + args.inputPaths.push_back( argv[argIndex] ); + } + if( args.inputPaths.size() == 0 or args.outputPath.empty() ) + { + Usage(); + } + if( std::filesystem::exists( args.outputPath ) and not overwrite ) + { + printf( "Output file %s already exists! Use -f to force overwrite.\n", args.outputPath.c_str() ); + exit( 4 ); + } + for( auto const& input : args.inputPaths ) + { + if( not std::filesystem::exists( input ) ) + { + printf( "Input file %s does not exist!\n", input.c_str() ); + exit( 4 ); + } + } + return args; + } +}; + +int main( int argc, char* argv[] ) +{ + auto args = Args::parse( argc, argv ); + + std::vector exports; + for( auto path : args.inputPaths ) + { + auto importedOpt = ExportedWorker::fromTracyFile( path, args.exportPlots ); + if( not importedOpt.has_value() ) + { + std::cerr << "Error importing " << path << std::endl; + return 1; + } + exports.push_back( ( importedOpt.value() ) ); + } + + std::vector exportRefs; + std::transform( exports.cbegin(), exports.cend(), std::back_inserter( exportRefs ), + []( ExportedWorker const& ex ) { return &ex; } ); + + auto mergedImport = ExportedWorker::merge( exportRefs ); + + { + std::cout << "Writing " << args.outputPath << std::endl; + auto outputFileWrite = std::unique_ptr( tracy::FileWrite::Open( args.outputPath.c_str() ) ); + if( !outputFileWrite ) + { + fprintf( stderr, "Cannot open output file!\n" ); + exit( 1 ); + } + tracy::Worker outputWorker( mergedImport.name.c_str(), mergedImport.process.c_str(), mergedImport.timeline, + mergedImport.messages, mergedImport.plots, mergedImport.threadNames ); + outputWorker.Write( *outputFileWrite, false ); + outputFileWrite->Finish(); + } + + std::cout << "done" << std::endl; + + return 0; +}