diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19d72ccf..19e5c047 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,7 @@ jobs: N_PROC: 8 Target: verilator REDMULE_COMPLEX: 0 + ProbStall: ${{ matrix.prob_stall }} Verilator: verilator VERILATOR_VERSION: "5.046" CCACHE_BASEDIR: ${{ github.workspace }} diff --git a/rtl/redmule_pkg.sv b/rtl/redmule_pkg.sv index 55c18c6b..8c94f031 100644 --- a/rtl/redmule_pkg.sv +++ b/rtl/redmule_pkg.sv @@ -60,7 +60,7 @@ package redmule_pkg; logic h_shift; logic load; logic pad_setup; - logic [$clog2(MaxDim)-1:0] width; + logic [$clog2(MaxDim):0] width; logic [$clog2(MaxDepth)-1:0] height; logic [$clog2(MaxDepth)-1:0] slots; logic rst_w_index; @@ -76,7 +76,7 @@ package redmule_pkg; logic shift; logic load; logic [$clog2(MaxDepth)-1:0] width; - logic [$clog2(MaxDim)-1:0] height; + logic [$clog2(MaxDim):0] height; logic [MaxDim-1:0] zero_set; } w_buffer_ctrl_t; @@ -93,10 +93,11 @@ package redmule_pkg; logic first_load; logic is_biased; logic mask_y; - logic [$clog2(MaxDim)-1:0] y_width; + logic [$clog2(MaxDim):0] y_width; logic [$clog2(MaxDepth)-1:0] y_height; - logic [$clog2(MaxDim)-1:0] z_width; + logic [$clog2(MaxDim):0] z_width; logic [$clog2(MaxDepth)-1:0] z_height; + logic [$clog2(MaxDim):0] z_rows; // valid (in-bounds) M rows of the tile being stored } z_buffer_ctrl_t; typedef struct packed { diff --git a/rtl/redmule_scheduler.sv b/rtl/redmule_scheduler.sv index 9c185171..90248b36 100644 --- a/rtl/redmule_scheduler.sv +++ b/rtl/redmule_scheduler.sv @@ -175,15 +175,19 @@ module redmule_scheduler assign x_done_en = /*flgs_streamer_i.x_stream_source_flags.ready_start &&*/ x_rows_iter_en && x_rows_iter_q == x_config.x_rows_iter-1 && x_w_iters_q == x_config.w_cols_iter-1 && x_cols_iter_q == x_config.x_cols_iter-1; - // For N <= Height, the "internal N" is promoted to a full N-tile (MinimumSizeN steps) by the tiler - // (see redmule_tiler.sv). The X buffer must then iterate a full D-deep tile so its refill / M-block - // row-advance stays in step with the promoted N; otherwise it would serve only - // x_buffer_slots (= n_size rounded up to Height, = Height for N <= Height) columns and never advance - // to the next M-block's X rows (the trailing output tile would reuse the first block's X). + // For N <= Height, the "internal N" is promoted to MinimumSizeN = MinimumSizeNFactor*Height W-row + // steps by the tiler (see redmule_tiler.sv). The X buffer must then iterate the same number of + // slots so its refill / M-block row-advance stays in step with the promoted N; otherwise it would + // serve only x_buffer_slots (= n_size rounded up to Height, = Height for N <= Height) columns and + // never advance to the next M-block's X rows (the trailing output tile would reuse the first + // block's X). The X tile cannot be deeper than D, so the promotion is capped there: for + // NumPipeRegs <= 1 this is exactly D, for NumPipeRegs > 1 it is MinimumSizeN (< D). + localparam int unsigned MinimumSizeN = MinimumSizeNFactor * H; + localparam int unsigned SmallNXSlots = (MinimumSizeN < D) ? MinimumSizeN : D; logic small_n_promoted_x; assign small_n_promoted_x = (x_config.n_size <= H); assign cntrl_x_buffer_o.height = x_cols_iter_q == x_config.x_cols_iter-1 && x_config.x_cols_lftovr != '0 ? x_config.x_cols_lftovr : D; - assign cntrl_x_buffer_o.slots = small_n_promoted_x ? D + assign cntrl_x_buffer_o.slots = small_n_promoted_x ? SmallNXSlots : (x_cols_iter_q == x_config.x_cols_iter-1 && x_config.x_cols_lftovr != '0 ? x_config.x_buffer_slots : D); assign cntrl_x_buffer_o.width = x_rows_iter_q == x_config.x_rows_iter-1 && x_config.x_rows_lftovr != '0 ? x_config.x_rows_lftovr : W; @@ -430,7 +434,7 @@ module redmule_scheduler z_avail_en, z_avail_clr, y_push_en, y_push_clr; - logic [$clog2(W):0] y_width, z_width; + logic [$clog2(W):0] y_width, z_width, z_rows; logic [$clog2(D):0] y_height, z_height; logic [$clog2(D):0] y_push_height; @@ -681,13 +685,18 @@ module redmule_scheduler assign y_push_counter_d = y_push_counter_q == y_push_height-1 ? '0 : y_push_counter_q + 1; assign y_push_clr = y_push_en && ~stall_engine && y_push_counter_q == y_push_height-1; - // A Y / Z tile is always W rows: the Y and Z streams move Width rows per row block (yz_tot_len) + // The Y load and Z store row counts are always W (d0_len = W in redmule_memory_scheduler.sv). + // The rows of the last z_rows_iter that fall outside Z are masked on the store side via + // z_rows (byte enables cleared in redmule_z_buffer.sv), so they never write past the end of Z. assign y_width = W; assign y_height = y_cols_iter_q == y_config_fast.w_cols_iter-1 && y_config_fast.w_cols_lftovr != '0 ? y_config_fast.w_cols_lftovr : D; - // Size of the tile in the Z buffer, tile (z_rows_iter_q, z_cols_iter_q) + // Size of the tile in the Z buffer, tile (z_rows_iter_q, z_cols_iter_q). Z is always stored + // W rows at a time; z_rows is the number of those rows that are inside Z (byte enables of + // the others are cleared in redmule_z_buffer.sv). assign z_width = W; assign z_height = z_cols_iter_q == z_config_fast.w_cols_iter-1 && z_config_fast.w_cols_lftovr != '0 ? z_config_fast.w_cols_lftovr : D; + assign z_rows = z_rows_iter_q == z_config_fast.x_rows_iter-1 && z_config_fast.x_rows_lftovr != '0 ? z_config_fast.x_rows_lftovr : W; assign cntrl_z_buffer_o.ready = z_ready_i; assign cntrl_z_buffer_o.y_valid = y_valid_i && y_config_fast.gemm_selection; @@ -701,6 +710,7 @@ module redmule_scheduler assign cntrl_z_buffer_o.y_height = y_push_height; assign cntrl_z_buffer_o.z_width = z_width; assign cntrl_z_buffer_o.z_height = z_height; + assign cntrl_z_buffer_o.z_rows = z_rows; /********************************** * Counters * diff --git a/rtl/x_buffer/redmule_x_buffer.sv b/rtl/x_buffer/redmule_x_buffer.sv index 15bd1b44..f9264a5c 100644 --- a/rtl/x_buffer/redmule_x_buffer.sv +++ b/rtl/x_buffer/redmule_x_buffer.sv @@ -270,8 +270,10 @@ always_ff @(posedge clk_i or negedge rst_ni) begin : buf_write_counter else begin if ((current_state == PAD_EMPTY || current_state == FAST_FILL && ~first_block) && ctrl_i.h_shift) buf_write_cnt <= buf_write_cnt + 1; + // The scm buffer feeds the array. It contains 2*H entries. In FAST_FILL we fill them up. + // The remaining entries (slots > 2*H) are written in FILL, one per h_shift. else if (ctrl_i.pad_setup) - buf_write_cnt <= ctrl_i.slots; + buf_write_cnt <= (ctrl_i.slots > 2*H) ? 2*H : ctrl_i.slots; end end end diff --git a/rtl/z_buffer/redmule_z_buffer.sv b/rtl/z_buffer/redmule_z_buffer.sv index 5365e4a5..122c062f 100644 --- a/rtl/z_buffer/redmule_z_buffer.sv +++ b/rtl/z_buffer/redmule_z_buffer.sv @@ -237,8 +237,12 @@ assign z_height = z_height_tmp[$clog2(D)-1:0]; always_comb begin : z_strb_assignment z_strb_o = '0; - for (int i = 0; i <= z_height; i++) begin - z_strb_o[i*BITW/8+:BITW/8] = '1; + // Z stream stores whole-Width tiles. + // Writes on partial output tiles (M % Width != 0) are masked by z_rows. + if (store_shift_q < ctrl_i.z_rows) begin + for (int i = 0; i <= z_height; i++) begin + z_strb_o[i*BITW/8+:BITW/8] = '1; + end end end diff --git a/scripts/ci-regression.sh b/scripts/ci-regression.sh index 666a4237..91272884 100755 --- a/scripts/ci-regression.sh +++ b/scripts/ci-regression.sh @@ -19,14 +19,59 @@ fi ScriptDir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BASE_TIMEOUT=500 +# Random memory stalls stretch the large tests well beyond the nominal runtime +# (e.g. M128_N128_K128 on 8x32x3 takes ~330-410 s at ProbStall=0.25). +if awk -v p="${ProbStall:-0}" 'BEGIN { exit !(p + 0 > 0) }'; then + BASE_TIMEOUT=900 +fi REGR_FILE="${REGR_FILE:-$ScriptDir/regression.yml}" # Number of tests to run concurrently. Each test now builds into its own # per-TEST_ID folder and only reads the shared, already-compiled RTL work # library, so the runs are isolated. Override N_PROC per invocation (bounded by # available Questa licenses / cores). N_PROC="${N_PROC:-4}" +REDMULE_PARAM_SETS="${REDMULE_PARAM_SETS:-8x8 8x32 8x32x3}" export Target -# be verbose in ci regression -python3 "$ScriptDir/bwruntests.py" -s --yaml -t $BASE_TIMEOUT -p "$N_PROC" --perf "$ScriptDir/perf.json" "$REGR_FILE" +status=0 + +for param_set in $REDMULE_PARAM_SETS; do + IFS=x read -r redmule_height redmule_width redmule_pipe_regs <