repo-sync-2024-09-25T14:14:57+0800 (#116)

* repo-sync-2024-09-25T14:14:57+0800

* fix header

* limit bazel build resource

* fix macos build

* fix build

* fix build

.bazelrc

@@ -14,11 +14,19 @@
 
 common --experimental_repo_remote_exec
 common --experimental_remote_download_regex='.*libserving\.so$|.*_pb2\.py$|.*\/secretflow_serving$|.*sf_serving\.tar\.gz$|.*\/simple_feature_service$'
+common --experimental_cc_shared_library
 
 build --incompatible_new_actions_api=false
 build --copt=-fdiagnostics-color=always
 build --enable_platform_specific_config
 
+# default off CUDA build
+build --@rules_cuda//cuda:enable=false
+
+# Only on when asked
+build:gpu --@rules_cuda//cuda:archs=compute_80:compute_80
+build:gpu --@rules_cuda//cuda:enable=true
+
 build --cxxopt=-std=c++17
 build --host_cxxopt=-std=c++17
 

.ci/accuracy_test.py

@@ -54,7 +54,7 @@ def dump_json(obj, filename, indent=2):
         json.dump(obj, ofile, indent=indent)
 
 
-def is_approximately_equal(a, b, epsilon) -> bool:
+def is_approximately_equal(a, b, epsilon=0.0001) -> bool:
     return abs(a - b) < epsilon
 
 
@@ -328,7 +328,7 @@ def _make_request_body(self):
 
         return json.dumps(body_dict)
 
-    def exec(self):
+    def exec(self, epsilon=0.0001):
         try:
             self.start_server()
 
@@ -364,7 +364,7 @@ def exec(self):
                         ]
                     )
                     assert is_approximately_equal(
-                        expect_score, s, 0.0001
+                        expect_score, s, epsilon
                     ), f'result not match, {s} vs {expect_score}'
         finally:
             self.stop_server()
@@ -459,3 +459,36 @@ def exec(self):
         query_ids=['1', '2', '3', '4', '5', '6', '7', '8', '9', '15'],
         score_col_name='pred',
     ).exec()
+
+    AccuracyTestCase(
+        service_id="phe_sgd",
+        parties=['alice', 'bob'],
+        case_dir='.ci/test_data/phe_sgd',
+        package_name='s_model.tar.gz',
+        input_csv_names={'alice': 'alice.csv', 'bob': 'bob.csv'},
+        expect_csv_name='predict.csv',
+        query_ids=['1', '2', '3', '4', '5', '6', '7', '8', '9', '15'],
+        score_col_name='pred',
+    ).exec()
+
+    AccuracyTestCase(
+        service_id="phe_sgd_no_feature",
+        parties=['alice', 'bob'],
+        case_dir='.ci/test_data/phe_sgd_no_feature',
+        package_name='s_model.tar.gz',
+        input_csv_names={'alice': 'alice.csv', 'bob': 'bob.csv'},
+        expect_csv_name='predict.csv',
+        query_ids=['1', '2', '3', '4', '5', '6', '7', '8', '9', '15'],
+        score_col_name='pred',
+    ).exec()
+
+    AccuracyTestCase(
+        service_id="phe_glm",
+        parties=['alice', 'bob'],
+        case_dir='.ci/test_data/phe_glm',
+        package_name='s_model.tar.gz',
+        input_csv_names={'alice': 'alice.csv', 'bob': 'bob.csv'},
+        expect_csv_name='predict.csv',
+        query_ids=['1', '2', '3', '4', '5', '6', '7', '8', '9', '15'],
+        score_col_name='predict_score',
+    ).exec(0.1)

.ci/inferencer_test.py

@@ -0,0 +1,78 @@
+# Copyright 2024 Ant Group Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from importlib import resources
+import asyncio
+import os
+
+current_file_path = os.path.abspath(__file__)
+code_dir = os.path.dirname(os.path.dirname(current_file_path))
+
+alice_serving_config_file_path = os.path.join(
+    code_dir,
+    "secretflow_serving/tools/inferencer/example/alice/serving.config",
+)
+alice_inference_config_file_path = os.path.join(
+    code_dir,
+    "secretflow_serving/tools/inferencer/example/alice/inference.config",
+)
+bob_serving_config_file_path = os.path.join(
+    code_dir,
+    "secretflow_serving/tools/inferencer/example/bob/serving.config",
+)
+bob_inference_config_file_path = os.path.join(
+    code_dir,
+    "secretflow_serving/tools/inferencer/example/bob/inference.config",
+)
+
+
+async def run_process(command):
+    process = await asyncio.create_subprocess_exec(
+        *command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+    )
+    stdout, stderr = await process.communicate()
+    if process.returncode == 0:
+        print(
+            f"Process {' '.join(command)} completed successfully:\n{stdout.decode().strip()}"
+        )
+    else:
+        print(
+            f"Process {' '.join(command)} failed with exit code {process.returncode}:\n{stderr.decode().strip()}"
+        )
+
+
+async def main():
+    with resources.path(
+        'secretflow_serving.tools.inferencer', 'inferencer'
+    ) as tool_path:
+        alice_command = [
+            str(tool_path),
+            f'--serving_config_file={alice_serving_config_file_path}',
+            f'--inference_config_file={alice_inference_config_file_path}',
+        ]
+        bob_command = [
+            str(tool_path),
+            f'--serving_config_file={bob_serving_config_file_path}',
+            f'--inference_config_file={bob_inference_config_file_path}',
+        ]
+        commands = [alice_command, bob_command]
+
+    tasks = [run_process(command) for command in commands]
+
+    await asyncio.gather(*tasks)
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

.ci/test_data/bin_onehot_glm/alice/alice.csv

@@ -18,4 +18,3 @@ id,f1,f2,f3,f4,f5,f6,f7,f8,b1,o1,y
 17,0.07245618290940148,0.15828585207480095,0.6058844851447147,-0.7307832385640813,-0.7047890815652045,0.7604535581160456,0.800552689601107,0.5620156263473772,0.18038921596661117,D,1.0
 18,0.9462315279587412,-0.16484385288965275,0.9134186991705504,-0.8007932195395477,0.9007963088557205,-0.2762376803205897,0.08557406326172878,0.2904407156393314,0.39477993406748413,D,0.0
 19,-0.24293124558329304,-0.6951771796317134,0.7522260924861339,-0.894295158530586,-0.14577787102095408,-0.2960307812622618,0.28186485638698366,-0.2308367011118997,0.014778052261847086,C,0.0
-20,0.104081262546454,-0.3402694280442802,0.6262946992127485,-0.8635426536480353,0.6833201820883588,0.011286885685380721,0.4785141057961586,0.9729862354474565,0.08367496429611809,D,0.0

.ci/test_data/phe_glm/alice/alice.csv

@@ -0,0 +1,33 @@
+id,f1,f2,f3,f4,b1,o1,y,unused1
+0,-0.1561563606294591,0.4091436724298469,0.7527352529453377,-0.4663496200949144,0.4168724773199035,C,1.0,0.6044702778742779
+1,-0.9404055611238592,-0.9083512326886756,-0.3706442384030441,0.2819235971596161,0.3517699625436855,D,0.0,0.7281280567505588
+2,-0.5627240503927933,-0.5442034486969063,0.3108773305897601,-0.7768956528082471,0.305838882862975,A,0.0,0.621498633148778
+3,0.0107105762067247,-0.4212240727957856,-0.2087361978786714,-0.13046949866179,0.4936165318157521,C,1.0,-0.4663885808110559
+4,-0.9469280606322728,-0.840416046152745,0.829095179481087,-0.0925525873415871,0.3269881588553663,C,1.0,0.5747490182709423
+5,-0.602324698626703,-0.5344182272779396,-0.0822962948252024,0.9076318550421604,0.0039115535760789,B,1.0,-0.7838087471940858
+6,0.2997688755590464,-0.7979971411805418,-0.4702396670038951,0.7517058807563881,0.4085520675577308,D,0.0,0.7443335658121795
+7,0.0898829612064333,-0.4440527937798157,-0.5067449846120331,-0.4732218984978185,0.1496893760999889,C,1.0,0.7171865026755633
+8,-0.5591187559186066,0.2713688885288003,0.1227362683263015,0.0011722261005966,0.3316943574830386,D,1.0,-0.5551325649086711
+9,0.1785313677518174,-0.2703356420598315,-0.4745167829541294,-0.6426962389397373,0.4694650019635519,D,0.0,0.6331732111938579
+10,0.6188609133556533,-0.2596380657662347,0.169171980447081,0.825255678689641,0.0671455571966838,D,1.0,-0.0793935306421158
+11,-0.987002480643878,-0.5809859384570246,0.795645767204954,0.7410371396735338,0.0577143352095511,B,1.0,-0.3896182653227988
+12,0.6116385036656158,-0.4660443559017733,-0.2011989897192054,-0.4031104171027342,0.0535179888547088,D,0.0,0.5906909983057236
+13,0.3962787899764537,0.873309175424988,-0.5613584816854333,0.2778989897320103,0.2766118204424079,B,1.0,-0.5448090251844593
+14,-0.3194989669640162,0.2960707704931871,0.9950752129902204,0.2179404228763446,0.1361741061574081,A,1.0,-0.952671130597097
+15,-0.6890410003764369,0.2182620113339763,0.019052587352929,-0.6943214629007304,0.3024149135151119,C,0.0,-0.6137404233445827
+16,0.9144261444135624,-0.657722703603806,-0.8181811756524122,0.5250216001503025,0.3588060935693989,C,0.0,-0.3434760976045869
+17,-0.3268109097747464,0.4582535959006983,-0.9057672491505308,0.0787580602392514,0.1017986561637264,A,1.0,0.7287058840605727
+18,-0.8145083132397042,-0.6731950124761432,-0.7807017392986817,0.5572529572611165,0.3171189794425398,B,1.0,0.9337782080967224
+19,-0.806567246333072,-0.2410891164847044,0.2548920834061801,0.0607073443903549,0.1319919508152047,C,1.0,-0.4417500145562572
+20,0.6949887326949196,0.9790467012731904,0.5841587287259282,-0.998856207744113,0.2442659260746882,B,1.0,0.2829634772152554
+21,0.2074520627337821,0.2799995197081857,-0.1556800664006319,-0.3516878859906538,0.4526682455396616,B,0.0,-0.2006432312798782
+22,0.6142565465487604,0.1138994875492924,-0.8729445876960857,-0.9610465152283354,0.4230518566474277,B,1.0,0.9622993743965202
+23,0.4594635733876357,0.3692285019797492,-0.2367614269869264,0.8581972325292342,0.0461492338563667,A,1.0,0.0724314649574437
+24,0.0724561829094014,0.6857038403796192,0.9922427604801936,0.7574437556463685,0.2117878862818631,A,1.0,0.8784742806494314
+25,0.9462315279587412,0.5519998230924896,0.058228690198274,0.6633310587223589,0.1383401119861258,B,1.0,-0.7693164962971448
+26,-0.242931245583293,-0.5419038560717913,0.9421567552272364,-0.3849717491946771,0.0017728445438911,D,0.0,0.940801222044456
+27,0.104081262546454,-0.9357995121919244,0.7215594044689961,-0.8841496670116249,0.3855596115098135,A,0.0,-0.6428643676550727
+28,0.6588093285059897,-0.3690939038818361,-0.9770379561143608,0.7560191984080811,0.3185566886506898,D,0.0,0.9250686315231108
+29,0.2370395047284921,-0.4645182480485945,0.4414436387203893,0.8938988905959881,0.1309776312171741,D,1.0,-0.4690672749540627
+30,0.7234138006215545,-0.5780343128273471,0.3634207380531495,-0.8286930958642424,0.3706154541739654,B,1.0,-0.7831949055705778
+31,0.154704290513524,0.8858194286701089,0.0739406608175903,-0.0280190733667724,0.2758402105631956,D,1.0,-0.1308724828707113

.ci/test_data/phe_glm/bob/bob.csv

@@ -0,0 +1,33 @@
+id,f5,f6,f7,f8,b2,o2,unused2
+0,-0.8615749630632328,-0.5715263852591228,0.9379925145695024,0.1900702129000553,0.2138434594903396,D,0.4570901213054086
+1,0.5212043305144631,-0.73537630254995,0.8527339660162552,0.3504251072081803,0.0048348498041699,D,-0.3726453716100175
+2,0.5316688586139755,0.871028481161342,0.6973914688286109,-0.5295922099981376,0.0376219300368835,B,0.2124177066122865
+3,-0.7432170710004744,0.1420861866505689,-0.667377778792172,-0.7602267721057516,0.4415531966500715,B,0.0228461193389561
+4,-0.0494352438025373,-0.0546579473764117,-0.028717749098563,0.780574628258875,0.4519642857799466,A,-0.2296091333105456
+5,0.0996071869898878,0.5692388485815068,-0.5725054016016367,-0.5075693044227503,0.2727951446027611,A,0.153176086993199
+6,-0.4698867421198818,0.6149939955332868,-0.1979194149010947,0.1890383070668824,0.4172975099430083,D,-0.4905549877228361
+7,0.7448660821705149,-0.6191801712762446,-0.8827292000556421,0.2387630206642061,0.291254783244897,B,0.4175705676683412
+8,-0.1537241195982261,-0.8061383715423533,-0.2420537620461678,-0.161550169328255,0.0740468927837413,B,-0.9966174435627412
+9,-0.5764035891158359,-0.1378976351872449,0.970617687559452,0.1673445785824494,0.0637227596410693,D,0.8511503309981654
+10,0.0785921775589166,-0.1528427539601584,-0.4695938836556961,0.0455654310639177,0.1541291749650668,A,0.0769039941855838
+11,0.4598621381799523,-0.06595066392665,0.5681412038971387,0.8694125154728545,0.449490744371295,B,0.438859998289691
+12,-0.5976978732206082,0.4581516989197012,-0.0899832653217134,-0.5914816011529271,0.3980611524440208,A,0.483900155678953
+13,-0.3765674173982101,0.346729094586603,-0.1539850280196741,0.4323836015788296,0.4303512910004514,D,0.3412570088659989
+14,0.9902987133217892,0.9683304227319324,0.9146352817193464,-0.522628094768308,0.4494623182632373,B,-0.2715570564374716
+15,0.299756115278907,-0.8031642576960822,0.9908453789854276,-0.208428306417491,0.105038269169877,D,-0.8600523777473796
+16,-0.1237998321709918,-0.1947574357954624,0.1115366468112364,0.3433804459199425,0.1247648696114622,C,0.3284753698225446
+17,0.0351516820711812,-0.3213947892100737,0.436816550592652,-0.4000058404024755,0.0513968108358928,D,-0.3395999279148072
+18,-0.7579916082634686,0.7233450727055821,-0.6904063494518717,-0.3676456074562919,0.3900581209357213,D,-0.3721687098832806
+19,-0.5506053259368853,-0.5026873321594287,-0.4065843490108716,0.5037289848288042,0.4420673507255044,D,0.696030559012671
+20,-0.3238288757050893,-0.619582183118377,0.9374187299383177,-0.8549137710136854,0.2031886949160584,B,0.4395085260279003
+21,0.1766174369144666,-0.1027729043337362,0.1583605816325124,-0.0834289547628277,0.3103307550753564,D,-0.3993554635774716
+22,-0.539770534806846,-0.1562367203311916,0.0843904027485484,0.9969088817088848,0.0772766691661023,D,-0.3814306755826935
+23,-0.559565231096881,-0.442909710666119,0.4959511207581282,0.9921928957101888,0.4649405078468372,C,-0.1832141827615663
+24,-0.8580138279819349,-0.500387104235799,-0.8856694541850338,-0.853478557800734,0.432302848109982,B,-0.1951992258845507
+25,0.2622059145401978,0.8465311985520256,0.1683551889179424,-0.5736913754659192,0.4881030164654814,C,-0.4086895949481059
+26,-0.5421164323776912,-0.113738509893086,0.0057007658390271,-0.4695991704991973,0.4053858599701984,D,-0.7454244018816936
+27,0.8108400260122559,0.7226982095236612,0.7054397840965707,0.8665187559874181,0.4407081023316622,B,-0.1591073324541834
+28,0.719270800507493,0.1006506248996961,-0.6851345441210335,0.7617283473728791,0.0123931809490943,C,0.880727341460366
+29,-0.8582853002226931,-0.8988233409502375,0.9215578065489008,0.7585404849690855,0.368282235877541,D,0.3546358905454658
+30,-0.5239907312620096,0.9985649368254532,-0.8397770695188262,-0.2609458225222321,0.1660927339732143,A,0.8056110914651653
+31,0.3379555565925611,0.6720551701599038,-0.6283500780385536,-0.6845063352855361,0.4654079430241628,C,0.231029831902761

.ci/test_data/phe_glm/predict.csv

@@ -0,0 +1,33 @@
+id,pred,y,predict_score
+0,0.47686505,1.0,516.2334798971686
+1,0.16220368,0.0,560.9372970797582
+2,0.26960367,0.0,542.3182257710636
+3,0.5657716,1.0,505.9261270108796
+4,0.17707317,1.0,557.889809489704
+5,0.27827334,1.0,541.0604345211162
+6,0.6727009,0.0,492.7743294058237
+7,0.58304656,1.0,503.8869425254623
+8,0.29150468,1.0,539.1862235038279
+9,0.5982337,0.0,502.07438558814715
+10,0.7883281,1.0,475.62210737259727
+11,0.17442402,1.0,558.4174829840767
+12,0.7779472,0.0,477.38605063822337
+13,0.7261041,1.0,485.4304663269026
+14,0.39520478,1.0,525.8383646190198
+15,0.23407295,0.0,547.7664128967078
+16,0.8454458,0.0,464.5294473247045
+17,0.36718696,1.0,529.2667277402132
+18,0.20782131,1.0,552.171084113332
+19,0.21610352,1.0,550.7402489345043
+20,0.8168119,1.0,470.4278724665369
+21,0.6409019,0.0,496.84692465294745
+22,0.7754037,1.0,477.8091725130769
+23,0.7400456,1.0,483.37432514560277
+24,0.63102967,1.0,498.07737805471197
+25,0.8610095,1.0,460.94058571637396
+26,0.42536157,0.0,522.2407333155013
+27,0.57785815,0.0,504.5016875280319
+28,0.77958846,0.0,477.111178457454
+29,0.66062826,1.0,494.3419931815475
+30,0.8147475,1.0,470.8242355794075
+31,0.6115879,1.0,500.4620059054651