fpga/common/XUPP3R: Add RX_CLK_FROM_TX configuration parameter to XUP…

…P3R 100G design Signed-off-by: Alex Forencich <[email protected]>
ucsdsysnet · Feb 17, 2024 · 9218698 · 9218698
1 parent 01c6d2f
commit 9218698
Show file tree

Hide file tree

Showing 6 changed files with 46 additions and 3 deletions.
diff --git a/fpga/mqnic/XUPP3R/fpga_100g/fpga/config.tcl b/fpga/mqnic/XUPP3R/fpga_100g/fpga/config.tcl
@@ -151,6 +151,7 @@ dict set params AXIS_ETH_TX_FIFO_PIPELINE "4"
 dict set params AXIS_ETH_TX_TS_PIPELINE "4"
 dict set params AXIS_ETH_RX_PIPELINE "4"
 dict set params AXIS_ETH_RX_FIFO_PIPELINE "4"
+dict set params ETH_RX_CLK_FROM_TX "0"
 
 # Statistics counter subsystem
 dict set params STAT_ENABLE "1"

diff --git a/fpga/mqnic/XUPP3R/fpga_100g/fpga_app_dma_bench/config.tcl b/fpga/mqnic/XUPP3R/fpga_100g/fpga_app_dma_bench/config.tcl
@@ -151,6 +151,7 @@ dict set params AXIS_ETH_TX_FIFO_PIPELINE "4"
 dict set params AXIS_ETH_TX_TS_PIPELINE "4"
 dict set params AXIS_ETH_RX_PIPELINE "4"
 dict set params AXIS_ETH_RX_FIFO_PIPELINE "4"
+dict set params ETH_RX_CLK_FROM_TX "0"
 
 # Statistics counter subsystem
 dict set params STAT_ENABLE "1"

diff --git a/fpga/mqnic/XUPP3R/fpga_100g/rtl/fpga.v b/fpga/mqnic/XUPP3R/fpga_100g/rtl/fpga.v
@@ -129,6 +129,7 @@ module fpga #
     parameter AXIS_ETH_TX_TS_PIPELINE = 4,
     parameter AXIS_ETH_RX_PIPELINE = 4,
     parameter AXIS_ETH_RX_FIFO_PIPELINE = 4,
+    parameter ETH_RX_CLK_FROM_TX = 0,
 
     // Statistics counter subsystem
     parameter STAT_ENABLE = 1,
@@ -1034,6 +1035,8 @@ wire                           qsfp0_rx_axis_tvalid_int;
 wire                           qsfp0_rx_axis_tlast_int;
 wire [80+1-1:0]                qsfp0_rx_axis_tuser_int;
 
+wire                           qsfp0_rx_ptp_clk_int;
+wire                           qsfp0_rx_ptp_rst_int;
 wire [79:0]                    qsfp0_rx_ptp_time_int;
 
 wire        qsfp0_drp_clk = clk_125mhz_int;
@@ -1101,6 +1104,7 @@ cmac_gty_wrapper #(
     .AXIS_KEEP_WIDTH(AXIS_ETH_KEEP_WIDTH),
     .TX_SERDES_PIPELINE(0),
     .RX_SERDES_PIPELINE(0),
+    .RX_CLK_FROM_TX(ETH_RX_CLK_FROM_TX),
     .RS_FEC_ENABLE(1)
 )
 qsfp0_cmac_inst (
@@ -1166,6 +1170,8 @@ qsfp0_cmac_inst (
     .rx_axis_tlast(qsfp0_rx_axis_tlast_int),
     .rx_axis_tuser(qsfp0_rx_axis_tuser_int),
 
+    .rx_ptp_clk(qsfp0_rx_ptp_clk_int),
+    .rx_ptp_rst(qsfp0_rx_ptp_rst_int),
     .rx_ptp_time(qsfp0_rx_ptp_time_int),
 
     .rx_enable(qsfp0_rx_enable),
@@ -1203,6 +1209,8 @@ wire                           qsfp1_rx_axis_tvalid_int;
 wire                           qsfp1_rx_axis_tlast_int;
 wire [80+1-1:0]                qsfp1_rx_axis_tuser_int;
 
+wire                           qsfp1_rx_ptp_clk_int;
+wire                           qsfp1_rx_ptp_rst_int;
 wire [79:0]                    qsfp1_rx_ptp_time_int;
 
 wire        qsfp1_drp_clk = clk_125mhz_int;
@@ -1270,6 +1278,7 @@ cmac_gty_wrapper #(
     .AXIS_KEEP_WIDTH(AXIS_ETH_KEEP_WIDTH),
     .TX_SERDES_PIPELINE(0),
     .RX_SERDES_PIPELINE(0),
+    .RX_CLK_FROM_TX(ETH_RX_CLK_FROM_TX),
     .RS_FEC_ENABLE(1)
 )
 qsfp1_cmac_inst (
@@ -1335,6 +1344,8 @@ qsfp1_cmac_inst (
     .rx_axis_tlast(qsfp1_rx_axis_tlast_int),
     .rx_axis_tuser(qsfp1_rx_axis_tuser_int),
 
+    .rx_ptp_clk(qsfp1_rx_ptp_clk_int),
+    .rx_ptp_rst(qsfp1_rx_ptp_rst_int),
     .rx_ptp_time(qsfp1_rx_ptp_time_int),
 
     .rx_enable(qsfp1_rx_enable),
@@ -1372,6 +1383,8 @@ wire                           qsfp2_rx_axis_tvalid_int;
 wire                           qsfp2_rx_axis_tlast_int;
 wire [80+1-1:0]                qsfp2_rx_axis_tuser_int;
 
+wire                           qsfp2_rx_ptp_clk_int;
+wire                           qsfp2_rx_ptp_rst_int;
 wire [79:0]                    qsfp2_rx_ptp_time_int;
 
 wire        qsfp2_drp_clk = clk_125mhz_int;
@@ -1439,6 +1452,7 @@ cmac_gty_wrapper #(
     .AXIS_KEEP_WIDTH(AXIS_ETH_KEEP_WIDTH),
     .TX_SERDES_PIPELINE(0),
     .RX_SERDES_PIPELINE(0),
+    .RX_CLK_FROM_TX(ETH_RX_CLK_FROM_TX),
     .RS_FEC_ENABLE(1)
 )
 qsfp2_cmac_inst (
@@ -1504,6 +1518,8 @@ qsfp2_cmac_inst (
     .rx_axis_tlast(qsfp2_rx_axis_tlast_int),
     .rx_axis_tuser(qsfp2_rx_axis_tuser_int),
 
+    .rx_ptp_clk(qsfp2_rx_ptp_clk_int),
+    .rx_ptp_rst(qsfp2_rx_ptp_rst_int),
     .rx_ptp_time(qsfp2_rx_ptp_time_int),
 
     .rx_enable(qsfp2_rx_enable),
@@ -1541,6 +1557,8 @@ wire                           qsfp3_rx_axis_tvalid_int;
 wire                           qsfp3_rx_axis_tlast_int;
 wire [80+1-1:0]                qsfp3_rx_axis_tuser_int;
 
+wire                           qsfp3_rx_ptp_clk_int;
+wire                           qsfp3_rx_ptp_rst_int;
 wire [79:0]                    qsfp3_rx_ptp_time_int;
 
 wire        qsfp3_drp_clk = clk_125mhz_int;
@@ -1608,6 +1626,7 @@ cmac_gty_wrapper #(
     .AXIS_KEEP_WIDTH(AXIS_ETH_KEEP_WIDTH),
     .TX_SERDES_PIPELINE(0),
     .RX_SERDES_PIPELINE(0),
+    .RX_CLK_FROM_TX(ETH_RX_CLK_FROM_TX),
     .RS_FEC_ENABLE(1)
 )
 qsfp3_cmac_inst (
@@ -1673,6 +1692,8 @@ qsfp3_cmac_inst (
     .rx_axis_tlast(qsfp3_rx_axis_tlast_int),
     .rx_axis_tuser(qsfp3_rx_axis_tuser_int),
 
+    .rx_ptp_clk(qsfp3_rx_ptp_clk_int),
+    .rx_ptp_rst(qsfp3_rx_ptp_rst_int),
     .rx_ptp_time(qsfp3_rx_ptp_time_int),
 
     .rx_enable(qsfp3_rx_enable),
@@ -2391,6 +2412,7 @@ fpga_core #(
     .AXIS_ETH_TX_TS_PIPELINE(AXIS_ETH_TX_TS_PIPELINE),
     .AXIS_ETH_RX_PIPELINE(AXIS_ETH_RX_PIPELINE),
     .AXIS_ETH_RX_FIFO_PIPELINE(AXIS_ETH_RX_FIFO_PIPELINE),
+    .ETH_RX_CLK_FROM_TX(ETH_RX_CLK_FROM_TX),
 
     // Statistics counter subsystem
     .STAT_ENABLE(STAT_ENABLE),
@@ -2536,6 +2558,8 @@ core_inst (
     .qsfp0_rx_axis_tvalid(qsfp0_rx_axis_tvalid_int),
     .qsfp0_rx_axis_tlast(qsfp0_rx_axis_tlast_int),
     .qsfp0_rx_axis_tuser(qsfp0_rx_axis_tuser_int),
+    .qsfp0_rx_ptp_clk(qsfp0_rx_ptp_clk_int),
+    .qsfp0_rx_ptp_rst(qsfp0_rx_ptp_rst_int),
     .qsfp0_rx_ptp_time(qsfp0_rx_ptp_time_int),
 
     .qsfp0_rx_enable(qsfp0_rx_enable),
@@ -2594,6 +2618,8 @@ core_inst (
     .qsfp1_rx_axis_tvalid(qsfp1_rx_axis_tvalid_int),
     .qsfp1_rx_axis_tlast(qsfp1_rx_axis_tlast_int),
     .qsfp1_rx_axis_tuser(qsfp1_rx_axis_tuser_int),
+    .qsfp1_rx_ptp_clk(qsfp1_rx_ptp_clk_int),
+    .qsfp1_rx_ptp_rst(qsfp1_rx_ptp_rst_int),
     .qsfp1_rx_ptp_time(qsfp1_rx_ptp_time_int),
 
     .qsfp1_rx_enable(qsfp1_rx_enable),
@@ -2652,6 +2678,8 @@ core_inst (
     .qsfp2_rx_axis_tvalid(qsfp2_rx_axis_tvalid_int),
     .qsfp2_rx_axis_tlast(qsfp2_rx_axis_tlast_int),
     .qsfp2_rx_axis_tuser(qsfp2_rx_axis_tuser_int),
+    .qsfp2_rx_ptp_clk(qsfp2_rx_ptp_clk_int),
+    .qsfp2_rx_ptp_rst(qsfp2_rx_ptp_rst_int),
     .qsfp2_rx_ptp_time(qsfp2_rx_ptp_time_int),
 
     .qsfp2_rx_enable(qsfp2_rx_enable),
@@ -2710,6 +2738,8 @@ core_inst (
     .qsfp3_rx_axis_tvalid(qsfp3_rx_axis_tvalid_int),
     .qsfp3_rx_axis_tlast(qsfp3_rx_axis_tlast_int),
     .qsfp3_rx_axis_tuser(qsfp3_rx_axis_tuser_int),
+    .qsfp3_rx_ptp_clk(qsfp3_rx_ptp_clk_int),
+    .qsfp3_rx_ptp_rst(qsfp3_rx_ptp_rst_int),
     .qsfp3_rx_ptp_time(qsfp3_rx_ptp_time_int),
 
     .qsfp3_rx_enable(qsfp3_rx_enable),

diff --git a/fpga/mqnic/XUPP3R/fpga_100g/rtl/fpga_core.v b/fpga/mqnic/XUPP3R/fpga_100g/rtl/fpga_core.v
@@ -151,6 +151,7 @@ module fpga_core #
     parameter AXIS_ETH_TX_TS_PIPELINE = 4,
     parameter AXIS_ETH_RX_PIPELINE = 4,
     parameter AXIS_ETH_RX_FIFO_PIPELINE = 4,
+    parameter ETH_RX_CLK_FROM_TX = 0,
 
     // Statistics counter subsystem
     parameter STAT_ENABLE = 1,
@@ -300,6 +301,8 @@ module fpga_core #
     input  wire                               qsfp0_rx_axis_tlast,
     input  wire [80+1-1:0]                    qsfp0_rx_axis_tuser,
 
+    input  wire                               qsfp0_rx_ptp_clk,
+    input  wire                               qsfp0_rx_ptp_rst,
     output wire [79:0]                        qsfp0_rx_ptp_time,
 
     output wire                               qsfp0_rx_enable,
@@ -362,6 +365,8 @@ module fpga_core #
     input  wire                               qsfp1_rx_axis_tlast,
     input  wire [80+1-1:0]                    qsfp1_rx_axis_tuser,
 
+    input  wire                               qsfp1_rx_ptp_clk,
+    input  wire                               qsfp1_rx_ptp_rst,
     output wire [79:0]                        qsfp1_rx_ptp_time,
 
     output wire                               qsfp1_rx_enable,
@@ -424,6 +429,8 @@ module fpga_core #
     input  wire                               qsfp2_rx_axis_tlast,
     input  wire [80+1-1:0]                    qsfp2_rx_axis_tuser,
 
+    input  wire                               qsfp2_rx_ptp_clk,
+    input  wire                               qsfp2_rx_ptp_rst,
     output wire [79:0]                        qsfp2_rx_ptp_time,
 
     output wire                               qsfp2_rx_enable,
@@ -486,6 +493,8 @@ module fpga_core #
     input  wire                               qsfp3_rx_axis_tlast,
     input  wire [80+1-1:0]                    qsfp3_rx_axis_tuser,
 
+    input  wire                               qsfp3_rx_ptp_clk,
+    input  wire                               qsfp3_rx_ptp_rst,
     output wire [79:0]                        qsfp3_rx_ptp_time,
 
     output wire                               qsfp3_rx_enable,
@@ -1263,8 +1272,8 @@ mqnic_port_map_mac_axis_inst (
     .mac_rx_clk({qsfp3_rx_clk, qsfp2_rx_clk, qsfp1_rx_clk, qsfp0_rx_clk}),
     .mac_rx_rst({qsfp3_rx_rst, qsfp2_rx_rst, qsfp1_rx_rst, qsfp0_rx_rst}),
 
-    .mac_rx_ptp_clk(4'b0000),
-    .mac_rx_ptp_rst(4'b0000),
+    .mac_rx_ptp_clk({qsfp3_rx_ptp_clk, qsfp2_rx_ptp_clk, qsfp1_rx_ptp_clk, qsfp0_rx_ptp_clk}),
+    .mac_rx_ptp_rst({qsfp3_rx_ptp_rst, qsfp2_rx_ptp_rst, qsfp1_rx_ptp_rst, qsfp0_rx_ptp_rst}),
     .mac_rx_ptp_ts_96({qsfp3_rx_ptp_time[47:0], qsfp2_rx_ptp_time[47:0], qsfp1_rx_ptp_time[47:0], qsfp0_rx_ptp_time[47:0]}),
     .mac_rx_ptp_ts_step(),
 
@@ -1365,7 +1374,7 @@ mqnic_core_pcie_us #(
     .PTP_CLOCK_PIPELINE(PTP_CLOCK_PIPELINE),
     .PTP_CLOCK_CDC_PIPELINE(PTP_CLOCK_CDC_PIPELINE),
     .PTP_SEPARATE_TX_CLOCK(0),
-    .PTP_SEPARATE_RX_CLOCK(0),
+    .PTP_SEPARATE_RX_CLOCK(ETH_RX_CLK_FROM_TX),
     .PTP_PORT_CDC_PIPELINE(PTP_PORT_CDC_PIPELINE),
     .PTP_PEROUT_ENABLE(PTP_PEROUT_ENABLE),
     .PTP_PEROUT_COUNT(PTP_PEROUT_COUNT),

diff --git a/fpga/mqnic/XUPP3R/fpga_100g/tb/fpga_core/Makefile b/fpga/mqnic/XUPP3R/fpga_100g/tb/fpga_core/Makefile
@@ -209,6 +209,7 @@ export PARAM_AXIS_ETH_TX_FIFO_PIPELINE := 4
 export PARAM_AXIS_ETH_TX_TS_PIPELINE := 4
 export PARAM_AXIS_ETH_RX_PIPELINE := 4
 export PARAM_AXIS_ETH_RX_FIFO_PIPELINE := 4
+export PARAM_ETH_RX_CLK_FROM_TX := 0
 
 # Statistics counter subsystem
 export PARAM_STAT_ENABLE := 1

diff --git a/fpga/mqnic/XUPP3R/fpga_100g/tb/fpga_core/test_fpga_core.py b/fpga/mqnic/XUPP3R/fpga_100g/tb/fpga_core/test_fpga_core.py
@@ -773,6 +773,7 @@ def test_fpga_core(request):
     parameters['AXIS_ETH_TX_TS_PIPELINE'] = 4
     parameters['AXIS_ETH_RX_PIPELINE'] = 4
     parameters['AXIS_ETH_RX_FIFO_PIPELINE'] = 4
+    parameters['ETH_RX_CLK_FROM_TX'] = 0
 
     # Statistics counter subsystem
     parameters['STAT_ENABLE'] = 1