Skip to content

Commit

Permalink
Merge pull request #783 from swaschkut/main
Browse files Browse the repository at this point in the history 
publish version 2.1.19
  • Loading branch information
@swaschkut
swaschkut authored Oct 6, 2023
2 parents ebf008e + 99c381e commit efc5861
Show file tree
Hide file tree
Showing 57 changed files with 3,650 additions and 291 deletions.
37 changes: 33 additions & 4 deletions CHANGELOG.txt
100644 → 100755
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,42 @@
CHANGELOG

2.1.18
2.1.19
UTIL:

* different classes | improve usage of flag 'XML_PARSE_BIG_LINES' in method loadXML
* class PANConf / Panoramaconfg | implement method ChildDeviceGroups() for 'location=shared:excludemaindg'
* type=gcp | introduce argument 'namespace=XYZ' to specify and better filter for get pods
* type=device | introduce actions=virtualSystem-delete/sharedgateway-delete/sharedgateway-migrate-to-vsys
* type=static-route | introduction of new type= | with 'filter=(nexthop-ip is.set)' / (nexthop-vr is.set) / nexthop-interface is.set) / (destination ip4.includes-full 172.33.10.0/23)
* type=static-route actions=delete | introduce new action
* type=rule | introduce actions=from-/to-remove-from-file:FILE.txt
* type=XYZ | introduce new arguments: shadow-loadreduce - to not update TAG object related address-groups | debugloadtime - display load time for specific XML sections
* develop | introduce config_validation.php
* type=appid-toolbox | improvements for further. new features - example rule address/service report generator
* type=rule actions=name-replace-character | set default value for replace to ''
* type=rule | introduce new actions=stats-appid-fastapi:-90days/stats-service-fastapi:/stats-address-source-fastapi/stats-address-destination-fastapi/stats-address-fastapi/stats-traffic-fastapi
* type=routing | introduce actions=exporttoexcel:file.html
* type=routing | introduce 'filter=(protocol.bgp is.enabled)'
* type=gcp | introdruce $namespace | improve handling for tenant like togglesvc
* type=xpath | introduce actions=remove | introduce 'filter-text=NODETEXT'

BUGFIX:
* class PANConf|VirtualSystem | bugfix to support SharedGateway also for Tag objects correctly
* type=xml-issue | bugfix to display read-only DeviceGroup duplicate address-group fixes correctly in summary
* type=address-merger | bugfix - do not merger address objects if tag count of planned merged object exceeds PAN-OS limit of 64 tag members
* type=address-merger | bugfix for removing tag objects from upper level, if adr merged objects are using them and tag is also available at address level
* bugfix for argument 'location=shared:excludemaindg' on FW config file
* type=schedule | bugfix actions=replacewithobject:OBJECTNAME

GENERAL:
* Class VirtualSystem/SharedGatewayStore | better handling of version variable
* update actions/filter JSON file
* APP-ID update to Device App-ID version: 8762-8327


2.1.18 (20230920)
BUGFIX:
* type=address/service 'actions=move:shared,skipIfConflict' | bugfix as variable $findSubSystem was not declared for targetlocation 'shared'

GENERAL:


2.1.17 (20230920)
UTIL:
Expand Down
152 changes: 121 additions & 31 deletions appid-toolbox/lib/common.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,20 +57,26 @@
class DeviceGroupRuleAppUsage
{
public $logs = Array();
public $logsSrcDst = Array();

public function load_from_file($filename)
public function load_from_file($filename, $SrcDst = false)
{
$xmlDoc = new DOMDocument();
$xmlDoc->Load($filename);

$recordsNode = DH::findFirstElementOrDie('records', $xmlDoc);

if( $SrcDst )
$keyword = "ips";
else
$keyword = "apps";

foreach( $recordsNode->childNodes as $entryNode )
{
if( $entryNode->nodeType != XML_ELEMENT_NODE )
continue;

$logRecord = Array( 'apps' => Array() );
$logRecord = Array( $keyword => Array() );

/** @var DOMElement $entryNode */

Expand All @@ -88,27 +94,55 @@ public function load_from_file($filename)

/** @var DOMElement $appNode */

$logRecord['apps'][$appNode->getAttribute('name')] = Array( 'name' => $appNode->getAttribute('name'), 'count' => $appNode->getAttribute('count'));
$logRecord[$keyword][$appNode->getAttribute('name')] = Array( 'name' => $appNode->getAttribute('name'), 'count' => $appNode->getAttribute('count'));
}

$this->logs[$ruleName] = &$logRecord;
if( $SrcDst )
$this->logsSrcDst[$ruleName] = &$logRecord;
else
$this->logs[$ruleName] = &$logRecord;
unset($logRecord);
}


}

public function save_to_file($filename)
public function save_to_file($filename, $SrcDst = false)
{
$xml = "<records>\n";

foreach($this->logs as $name => &$log)
if($SrcDst)
{
$logArray = $this->logsSrcDst;
$keyWord = "ips";
}
else
{
$logArray = $this->logs;
$keyWord = "apps";
}


foreach($logArray as $name => &$log)
{
$xml .= " <entry name=\"{$name}\" timestamp=\"{$log['timestamp']}\" Htimestamp=\"".timestamp_to_date($log['timestamp'])."\">\n";

foreach( $log['apps'] as &$app )
foreach( $log[$keyWord] as $key => &$app )
{
$xml .= " <app name=\"{$app['name']}\" count=\"{$app['count']}\"/>\n";
if( $keyWord == "apps")
{
$xml .= " <".$keyWord." name=\"{$app['name']}\" count=\"{$app['count']}\"/>\n";
}
else
{
#$xml .= " <".$keyWord.">\n";
foreach( $app as $ip )
{
$xml .= " <".$keyWord."-".$key." name=\"{$ip['name']}\" count=\"{$ip['count']}\"/>\n";
}

#$xml .= " </".$keyWord.">\n";
}
}

$xml .= " </entry>\n";
Expand All @@ -119,6 +153,7 @@ public function save_to_file($filename)
file_put_contents($filename, $xml);
}


public function addRuleStats($ruleName , $appName, $hitCount)
{
if( isset($this->logs[$ruleName]) )
Expand All @@ -139,33 +174,75 @@ public function addRuleStats($ruleName , $appName, $hitCount)
$record['apps'][$appName] = Array('name'=>$appName, 'count' => $hitCount);
}

public function addRuleStats_SrcDst($ruleName , $srcOrDst, $ip, $hitCount)
{
if( isset($this->logsSrcDst[$ruleName]) )
{
$record = &$this->logsSrcDst[$ruleName];
}
else
{
$SrcDstArray = array('src', 'dst');
$record = Array( 'ips' => $SrcDstArray );
$this->logsSrcDst[$ruleName] = &$record;
}

$record['timestamp'] = time();

if( isset($record['ips'][$srcOrDst][$ip]) )
$record['ips'][$srcOrDst][$ip]['count'] += $hitCount;
else
$record['ips'][$srcOrDst][$ip] = Array('name'=>$ip, 'count' => $hitCount);
}

/**
* @param string $ruleName
* @return null|int
*/
public function getRuleUpdateTimestamp($ruleName)
public function getRuleUpdateTimestamp($ruleName, $SrcDst = false)
{
if( isset($this->logs[$ruleName]) )
{
return $this->logs[$ruleName]['timestamp'];
}
if( $SrcDst )
if( isset($this->logsSrcDst[$ruleName]) )
{
return $this->logsSrcDst[$ruleName]['timestamp'];
}
else
if( isset($this->logs[$ruleName]) )
{
return $this->logs[$ruleName]['timestamp'];
}

return null;
}


public function resetRulesStats($ruleName)
public function resetRulesStats($ruleName, $SrcDst = false)
{
if( isset($this->logs[$ruleName]) )
unset($this->logs[$ruleName]);
if( $SrcDst )
if( isset($this->logsSrcDst[$ruleName]) )
unset($this->logsSrcDst[$ruleName]);
else
if( isset($this->logs[$ruleName]) )
unset($this->logs[$ruleName]);
}


public function getRuleStats($ruleName)
public function getRuleStats($ruleName, $SrcDst = false)
{
if( !isset($this->logs[$ruleName]) )
return null;
if( $SrcDst )
{
if( !isset($this->logsSrcDst[$ruleName]) )
return null;

return $this->logsSrcDst[$ruleName]['ips'];
}
else
{
if( !isset($this->logs[$ruleName]) )
return null;

return $this->logs[$ruleName]['apps'];
return $this->logs[$ruleName]['apps'];
}
}

public function isRuleUsed($ruleName, $ignoreApps = Array('incomplete', 'non-syn-tcp') )
Expand All @@ -183,21 +260,34 @@ public function isRuleUsed($ruleName, $ignoreApps = Array('incomplete', 'non-syn

}

public function createRuleStats($ruleName)
public function createRuleStats($ruleName, $SrcDst = false)
{
if( !isset($this->logs[$ruleName]) )
{
$record = Array( 'apps' => Array(), 'timestamp' => time() );
$this->logs[$ruleName] = &$record;
}
if( $SrcDst )
if( !isset($this->logsSrcDst[$ruleName]) )
{
$record = Array( 'ips' => Array(), 'timestamp' => time() );
$this->logsSrcDst[$ruleName] = &$record;
}
else
if( !isset($this->logs[$ruleName]) )
{
$record = Array( 'apps' => Array(), 'timestamp' => time() );
$this->logs[$ruleName] = &$record;
}
}

public function updateRuleUpdateTimestamp($ruleName)
public function updateRuleUpdateTimestamp($ruleName, $SrcDst = false)
{
if( isset($this->logs[$ruleName]) )
{
$this->logs[$ruleName]['timestamp'] = time();
}
if( $SrcDst )
if( isset($this->logsSrcDst[$ruleName]) )
{
$this->logsSrcDst[$ruleName]['timestamp'] = time();
}
else
if( isset($this->logs[$ruleName]) )
{
$this->logs[$ruleName]['timestamp'] = time();
}
}


Expand Down
15 changes: 10 additions & 5 deletions appid-toolbox/lib/trait/lib_1_rule_marker.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ function ruleMarker_Phase1_init()
if( isset(PH::$args['help']) )
$this->display_usage_and_exit_p1();

$supportedOptions = array('phase', 'in', 'out', 'help', 'location');
$supportedOptions = array('phase', 'in', 'out', 'help', 'location', 'debugapi');
$supportedOptions = array_flip($supportedOptions);

foreach( PH::$args as $arg => $argvalue )
Expand All @@ -54,6 +54,10 @@ function ruleMarker_Phase1_init()

$debugAPI = FALSE;

if( isset(PH::$args['debugapi']) )
{
$debugAPI = TRUE;
}

$return = AppIDToolbox_common::location();
$configInput = $return['configInput'];
Expand Down Expand Up @@ -99,11 +103,12 @@ function ruleMarker_Phase1_main($subSystem, $configInput, $pan, $inputConnector,

foreach( $rules as $rule )
{
PH::print_stdout();
PH::print_stdout(" - rule '{$rule->name()}'");

if( $ridTagLibrary->ruleIsTagged($rule) )
{
PH::print_stdout(" SKIPPED : already tagged");
PH::print_stdout(" SKIPPED : already tagged");
$alreadyMarked++;
continue;
}
Expand All @@ -112,7 +117,7 @@ function ruleMarker_Phase1_main($subSystem, $configInput, $pan, $inputConnector,


$newTagName = $ridTagLibrary->findAvailableTagName('appRID#');
PH::print_stdout();

PH::print_stdout(" * creating Virtual TAG '$newTagName' ... ");

PH::print_stdout(" * applying tag to rule description... ");
Expand All @@ -129,10 +134,10 @@ function ruleMarker_Phase1_main($subSystem, $configInput, $pan, $inputConnector,
$xmlPreRules .= "<entry name=\"{$rule->name()}\"><description>" . htmlspecialchars($rule->description()) . "</description></entry>";
}

PH::print_stdout("\n\nNumber of rules marked: {$markedRules} (vs already marked: {$alreadyMarked}");
PH::print_stdout("\n\nNumber of rules marked: '{$markedRules}' (vs already marked: '{$alreadyMarked}')");

if( $markedRules < 1 )
PH::print_stdout("\n\n No change to push as not rule is set to be marked");
PH::print_stdout("\n\nNo change to push as no rule is set to be marked");
else
{
if( $configInput['type'] == 'api' )
Expand Down
Loading

0 comments on commit efc5861

Please sign in to comment.