revised reflection handling

Instead of excluding specific names from obfuscation "all" names are now  obfuscated.

For reflected names, a mapping to the original name is injected in internal/abi to resolve them correctly.

Fixes #884, #799, #817, #881, #858, #843, #842

Closes #406

.gitignore

@@ -1,3 +1,4 @@
 /garble
 /test
 /bincmp_output/
+debug

AUTHORS

@@ -11,8 +11,8 @@ Daniel Martí <[email protected]>
 Emmanuel Chee-zaram Okeke <[email protected]>
 NHAS <[email protected]>
 Nicholas Jones <[email protected]>
+Paul Scheduikat <[email protected]>
 Zachary Wasserman <[email protected]>
-lu4p <[email protected]>
 pagran <[email protected]>
 shellhazard <[email protected]>
 xuannv <[email protected]>

README.md

@@ -157,23 +157,7 @@ to document the current shortcomings of this tool.
   be required by interfaces. This area is a work in progress; see
   [#3](https://github.com/burrowers/garble/issues/3).
 
-* Garble automatically detects which Go types are used with reflection
-  to avoid obfuscating them, as that might break your program.
-  Note that Garble obfuscates [one package at a time](#speed),
-  so if your reflection code inspects a type from an imported package,
-  you may need to add a "hint" in the imported package to exclude obfuscating it:
-   ```go
-   type Message struct {
-       Command string
-       Args    string
-   }
-
-   // Never obfuscate the Message type.
-   var _ = reflect.TypeOf(Message{})
-   ```
-
 * Aside from `GOGARBLE` to select patterns of packages to obfuscate,
-  and the hint above with `reflect.TypeOf` to exclude obfuscating particular types,
   there is no supported way to exclude obfuscating a selection of files or packages.
   More often than not, a user would want to do this to work around a bug; please file the bug instead.
 

hash.go

@@ -223,15 +223,7 @@ func entryOffKey() uint32 {
 	return runtimeHashWithCustomSalt([]byte("entryOffKey"))
 }
 
-func hashWithPackage(tf *transformer, pkg *listedPackage, name string) string {
-
-	// In some places it is not appropriate to access the transformer
-	if tf != nil {
-		// If the package is marked as "in-use" by reflection, the private structures are not obfuscated, so dont return them as a hash. Fixes #882
-		if _, ok := tf.curPkgCache.ReflectObjects[pkg.ImportPath+"."+name]; ok {
-			return name
-		}
-	}
+func hashWithPackage(pkg *listedPackage, name string) string {
 	// If the user provided us with an obfuscation seed,
 	// we use that with the package import path directly..
 	// Otherwise, we use GarbleActionID as a fallback salt.
@@ -348,8 +340,7 @@ func hashWithStruct(strct *types.Struct, field *types.Var) string {
 // For example, increasing the average length of 9 by 1 results in roughly a 1%
 // increase in binary sizes.
 const (
-	minHashLength = 6
-	maxHashLength = 12
+	hashLength = 9
 
 	// At most we'll need maxHashLength base64 characters,
 	// so 9 checksum bytes are enough for that purpose,
@@ -377,14 +368,6 @@ func hashWithCustomSalt(salt []byte, name string) string {
 	io.WriteString(hasher, name)
 	sum := hasher.Sum(sumBuffer[:0])
 
-	// The byte after neededSumBytes is never used as part of the name,
-	// but it is still deterministic and hard to predict,
-	// so it provides us with useful randomness between 0 and 255.
-	// We want the number to be between 0 and hashLenthRange-1 as well,
-	// so we use a remainder operation.
-	hashLengthRandomness := sum[neededSumBytes] % ((maxHashLength - minHashLength) + 1)
-	hashLength := minHashLength + hashLengthRandomness
-
 	nameBase64.Encode(b64NameBuffer[:], sum[:neededSumBytes])
 	b64Name := b64NameBuffer[:hashLength]
 
@@ -412,11 +395,9 @@ func hashWithCustomSalt(salt []byte, name string) string {
 				// Turn "afoo" into "Afoo".
 				b64Name[0] = toUpper(b64Name[0])
 			}
-		} else {
-			if isUpper(b64Name[0]) {
-				// Turn "Afoo" into "afoo".
-				b64Name[0] = toLower(b64Name[0])
-			}
+		} else if isUpper(b64Name[0]) {
+			// Turn "Afoo" into "afoo".
+			b64Name[0] = toLower(b64Name[0])
 		}
 	}
 	return string(b64Name)

main.go

@@ -696,7 +696,7 @@ func (tf *transformer) transformAsm(args []string) ([]string, error) {
 	newPaths := make([]string, 0, len(paths))
 	if !slices.Contains(args, "-gensymabis") {
 		for _, path := range paths {
-			name := hashWithPackage(tf, tf.curPkg, filepath.Base(path)) + ".s"
+			name := hashWithPackage(tf.curPkg, filepath.Base(path)) + ".s"
 			pkgDir := filepath.Join(sharedTempDir, tf.curPkg.obfuscatedImportPath())
 			newPath := filepath.Join(pkgDir, name)
 			newPaths = append(newPaths, newPath)
@@ -786,7 +786,7 @@ func (tf *transformer) transformAsm(args []string) ([]string, error) {
 		// directory, as assembly files do not support `/*line` directives.
 		// TODO(mvdan): per cmd/asm/internal/lex, they do support `#line`.
 		basename := filepath.Base(path)
-		newName := hashWithPackage(tf, tf.curPkg, basename) + ".s"
+		newName := hashWithPackage(tf.curPkg, basename) + ".s"
 		if path, err := tf.writeSourceFile(basename, newName, buf.Bytes()); err != nil {
 			return nil, err
 		} else {
@@ -902,7 +902,7 @@ func (tf *transformer) replaceAsmNames(buf *bytes.Buffer, remaining []byte) {
 		remaining = remaining[nameEnd:]
 
 		if lpkg.ToObfuscate && !compilerIntrinsics[lpkg.ImportPath][name] {
-			newName := hashWithPackage(tf, lpkg, name)
+			newName := hashWithPackage(lpkg, name)
 			if flagDebug { // TODO(mvdan): remove once https://go.dev/issue/53465 if fixed
 				log.Printf("asm name %q hashed with %x to %q", name, tf.curPkg.GarbleActionID, newName)
 			}
@@ -949,16 +949,39 @@ func (tf *transformer) writeSourceFile(basename, obfuscated string, content []by
 // parseFiles parses a list of Go files.
 // It supports relative file paths, such as those found in listedPackage.CompiledGoFiles,
 // as long as dir is set to listedPackage.Dir.
-func parseFiles(dir string, paths []string) ([]*ast.File, error) {
-	var files []*ast.File
+func parseFiles(lpkg *listedPackage, dir string, paths []string) (files []*ast.File, err error) {
+	mainPackage := lpkg.Name == "main" && lpkg.ForTest == ""
+
 	for _, path := range paths {
 		if !filepath.IsAbs(path) {
 			path = filepath.Join(dir, path)
 		}
-		file, err := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution|parser.ParseComments)
+
+		var src any
+
+		if lpkg.ImportPath == "internal/abi" && filepath.Base(path) == "type.go" {
+			src, err = abiNamePatch(path)
+			if err != nil {
+				return nil, err
+			}
+		} else if mainPackage && reflectPatchFile == "" {
+			src, err = reflectMainPrePatch(path)
+			if err != nil {
+				return nil, err
+			}
+
+			reflectPatchFile = path
+		}
+
+		file, err := parser.ParseFile(fset, path, src, parser.SkipObjectResolution|parser.ParseComments)
 		if err != nil {
 			return nil, err
 		}
+
+		if mainPackage && src != nil {
+			astutil.AddNamedImport(fset, file, "_", "unsafe")
+		}
+
 		files = append(files, file)
 	}
 	return files, nil
@@ -972,7 +995,7 @@ func (tf *transformer) transformCompile(args []string) ([]string, error) {
 	flags = append(flags, "-dwarf=false")
 
 	// The Go file paths given to the compiler are always absolute paths.
-	files, err := parseFiles("", paths)
+	files, err := parseFiles(tf.curPkg, "", paths)
 	if err != nil {
 		return nil, err
 	}
@@ -1026,8 +1049,12 @@ func (tf *transformer) transformCompile(args []string) ([]string, error) {
 		}
 	}
 
-	if tf.curPkgCache, err = loadPkgCache(tf.curPkg, tf.pkg, files, tf.info, ssaPkg); err != nil {
-		return nil, err
+	tf.curPkgCache, err = loadPkgCache(tf.curPkg)
+	if err != nil {
+		tf.curPkgCache, err = computePkgCache(tf.curPkg, tf.pkg, files, tf.info, ssaPkg)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	// These maps are not kept in pkgCache, since they are only needed to obfuscate curPkg.
@@ -1085,6 +1112,10 @@ func (tf *transformer) transformCompile(args []string) ([]string, error) {
 			return nil, err
 		}
 
+		if tf.curPkg.Name == "main" && strings.HasSuffix(reflectPatchFile, basename) {
+			src = reflectMainPostPatch(src, tf.curPkg)
+		}
+
 		// We hide Go source filenames via "//line" directives,
 		// so there is no need to use obfuscated filenames here.
 		if path, err := tf.writeSourceFile(basename, basename, src); err != nil {
@@ -1140,7 +1171,7 @@ func (tf *transformer) transformDirectives(comments []*ast.CommentGroup) {
 func (tf *transformer) transformLinkname(localName, newName string) (string, string) {
 	// obfuscate the local name, if the current package is obfuscated
 	if tf.curPkg.ToObfuscate && !compilerIntrinsics[tf.curPkg.ImportPath][localName] {
-		localName = hashWithPackage(tf, tf.curPkg, localName)
+		localName = hashWithPackage(tf.curPkg, localName)
 	}
 	if newName == "" {
 		return localName, ""
@@ -1208,29 +1239,26 @@ func (tf *transformer) transformLinkname(localName, newName string) (string, str
 
 	var newForeignName string
 	if receiver, name, ok := strings.Cut(foreignName, "."); ok {
-		if lpkg.ImportPath == "reflect" && (receiver == "(*rtype)" || receiver == "Value") {
-			// These receivers are not obfuscated.
-			// See the TODO below.
-		} else if strings.HasPrefix(receiver, "(*") {
+		if strings.HasPrefix(receiver, "(*") {
 			// pkg/path.(*Receiver).method
 			receiver = strings.TrimPrefix(receiver, "(*")
 			receiver = strings.TrimSuffix(receiver, ")")
-			receiver = "(*" + hashWithPackage(tf, lpkg, receiver) + ")"
+			receiver = "(*" + hashWithPackage(lpkg, receiver) + ")"
 		} else {
 			// pkg/path.Receiver.method
-			receiver = hashWithPackage(tf, lpkg, receiver)
+			receiver = hashWithPackage(lpkg, receiver)
 		}
 		// Exported methods are never obfuscated.
 		//
 		// TODO(mvdan): We're duplicating the logic behind these decisions.
 		// Reuse the logic with transformCompile.
 		if !token.IsExported(name) {
-			name = hashWithPackage(tf, lpkg, name)
+			name = hashWithPackage(lpkg, name)
 		}
 		newForeignName = receiver + "." + name
 	} else {
 		// pkg/path.function
-		newForeignName = hashWithPackage(tf, lpkg, foreignName)
+		newForeignName = hashWithPackage(lpkg, foreignName)
 	}
 
 	newPkgPath := lpkg.ImportPath
@@ -1308,7 +1336,7 @@ func (tf *transformer) processImportCfg(flags []string, requiredPkgs []string) (
 			// For beforePath="vendor/foo", afterPath and
 			// lpkg.ImportPath can be just "foo".
 			// Don't use obfuscatedImportPath here.
-			beforePath = hashWithPackage(tf, lpkg, beforePath)
+			beforePath = hashWithPackage(lpkg, beforePath)
 
 			afterPath = lpkg.obfuscatedImportPath()
 		}
@@ -1405,14 +1433,9 @@ type pkgCache struct {
 	// unless we were smart enough to detect which arguments get used as %#v or %T.
 	ReflectAPIs map[funcFullName]map[int]bool
 
-	// ReflectObjects is filled with the fully qualified names from each
-	// package that we cannot obfuscate due to reflection.
-	// The included objects are named types and their fields,
-	// since it is those names being obfuscated that could break the use of reflect.
-	//
-	// This record is necessary for knowing what names from imported packages
-	// weren't obfuscated, so we can obfuscate their local uses accordingly.
-	ReflectObjects map[objectString]struct{}
+	// ReflectObjectNames maps obfuscated names which are reflected to their "real"
+	// non-obfuscated names.
+	ReflectObjectNames map[objectString]string
 
 	// EmbeddedAliasFields records which embedded fields use a type alias.
 	// They are the only instance where a type alias matters for obfuscation,
@@ -1425,7 +1448,7 @@ type pkgCache struct {
 
 func (c *pkgCache) CopyFrom(c2 pkgCache) {
 	maps.Copy(c.ReflectAPIs, c2.ReflectAPIs)
-	maps.Copy(c.ReflectObjects, c2.ReflectObjects)
+	maps.Copy(c.ReflectObjectNames, c2.ReflectObjectNames)
 	maps.Copy(c.EmbeddedAliasFields, c2.EmbeddedAliasFields)
 }
 
@@ -1460,7 +1483,7 @@ func openCache() (*cache.Cache, error) {
 	return cache.Open(dir)
 }
 
-func loadPkgCache(lpkg *listedPackage, pkg *types.Package, files []*ast.File, info *types.Info, ssaPkg *ssa.Package) (pkgCache, error) {
+func loadPkgCache(lpkg *listedPackage) (pkgCache, error) {
 	fsCache, err := openCache()
 	if err != nil {
 		return pkgCache{}, err
@@ -1479,10 +1502,16 @@ func loadPkgCache(lpkg *listedPackage, pkg *types.Package, files []*ast.File, in
 		}
 		return loaded, nil
 	}
-	return computePkgCache(fsCache, lpkg, pkg, files, info, ssaPkg)
+
+	return pkgCache{}, fmt.Errorf("pkg: %s not cached yet", lpkg.ImportPath)
 }
 
-func computePkgCache(fsCache *cache.Cache, lpkg *listedPackage, pkg *types.Package, files []*ast.File, info *types.Info, ssaPkg *ssa.Package) (pkgCache, error) {
+func computePkgCache(lpkg *listedPackage, pkg *types.Package, files []*ast.File, info *types.Info, ssaPkg *ssa.Package) (pkgCache, error) {
+	fsCache, err := openCache()
+	if err != nil {
+		return pkgCache{}, err
+	}
+
 	// Not yet in the cache. Load the cache entries for all direct dependencies,
 	// build our cache entry, and write it to disk.
 	// Note that practically all errors from Cache.GetFile are a cache miss;
@@ -1497,7 +1526,7 @@ func computePkgCache(fsCache *cache.Cache, lpkg *listedPackage, pkg *types.Packa
 			"reflect.TypeOf":  {0: true},
 			"reflect.ValueOf": {0: true},
 		},
-		ReflectObjects:      map[objectString]struct{}{},
+		ReflectObjectNames:  map[objectString]string{},
 		EmbeddedAliasFields: map[objectString]typeName{},
 	}
 	for _, imp := range lpkg.Imports {
@@ -1530,7 +1559,7 @@ func computePkgCache(fsCache *cache.Cache, lpkg *listedPackage, pkg *types.Packa
 			// Missing or corrupted entry in the cache for a dependency.
 			// Could happen if GARBLE_CACHE was emptied but GOCACHE was not.
 			// Compute it, which can recurse if many entries are missing.
-			files, err := parseFiles(lpkg.Dir, lpkg.CompiledGoFiles)
+			files, err := parseFiles(lpkg, lpkg.Dir, lpkg.CompiledGoFiles)
 			if err != nil {
 				return err
 			}
@@ -1539,7 +1568,7 @@ func computePkgCache(fsCache *cache.Cache, lpkg *listedPackage, pkg *types.Packa
 			if err != nil {
 				return err
 			}
-			computedImp, err := computePkgCache(fsCache, lpkg, pkg, files, info, nil)
+			computedImp, err := computePkgCache(lpkg, pkg, files, info, nil)
 			if err != nil {
 				return err
 			}
@@ -1997,11 +2026,6 @@ func (tf *transformer) transformGoFile(file *ast.File) *ast.File {
 			}
 		}
 
-		// The package that declared this object did not obfuscate it.
-		if usedForReflect(tf.curPkgCache, obj) {
-			return true
-		}
-
 		lpkg, err := listPackage(tf.curPkg, path)
 		if err != nil {
 			panic(err) // shouldn't happen
@@ -2071,7 +2095,7 @@ func (tf *transformer) transformGoFile(file *ast.File) *ast.File {
 			return true // we only want to rename the above
 		}
 
-		node.Name = hashWithPackage(tf, lpkg, name)
+		node.Name = hashWithPackage(lpkg, name)
 		// TODO: probably move the debugf lines inside the hash funcs
 		if flagDebug { // TODO(mvdan): remove once https://go.dev/issue/53465 if fixed
 			log.Printf("%s %q hashed with %x… to %q", debugName, name, hashToUse[:4], node.Name)
@@ -2192,7 +2216,7 @@ func (tf *transformer) transformLink(args []string) ([]string, error) {
 		if path != "main" {
 			newPath = lpkg.obfuscatedImportPath()
 		}
-		newName := hashWithPackage(tf, lpkg, name)
+		newName := hashWithPackage(lpkg, name)
 		flags = append(flags, fmt.Sprintf("-X=%s.%s=%s", newPath, newName, stringValue))
 	})
 

position.go

@@ -127,7 +127,7 @@ func printFile(lpkg *listedPackage, file *ast.File) ([]byte, error) {
 			newName := ""
 			if !flagTiny {
 				origPos := fmt.Sprintf("%s:%d", filename, origOffset)
-				newName = hashWithPackage(nil, lpkg, origPos) + ".go"
+				newName = hashWithPackage(lpkg, origPos) + ".go"
 				// log.Printf("%q hashed with %x to %q", origPos, curPkg.GarbleActionID, newName)
 			}