bigcode/commitpack

1#!/bin/bash2 3input_file="diffs_33554432_41943040.jsonl"4input_file="diffs_58720256_67108864.jsonl"5input_file="diffs_0_8388608.jsonl"6total_lines=83886087num_shards=168 9lines_per_shard=$((total_lines / num_shards))10start_line=111 12for i in $(seq 1 $num_shards); do13    end_line=$((start_line + lines_per_shard - 1))14    start_range=$((start_line + 0))15    end_range=$((end_line + 0))16    output_file="diffs_${start_range}_${end_range}.jsonl"17    sed -n "${start_line},${end_line}p" $input_file > $output_file18    start_line=$((end_line + 1))19done20 21