check-memory (5230B)
1 #!/bin/sh 2 3 # NB: This program is a work in progress. 4 5 # This program invokes unison in varying ways to attempt to determine 6 # how much memory is required for a given sync, or alternatively how 7 # big a sync can be done in a given amount of memory. 8 9 # The program expects to own $HOME/UNISON-TEST, but tries not to 10 # damage any files existing when it is run. 11 12 # This program is written in POSIX /bin/sh and intends to use only 13 # tools specified by POSIX, specifically avoiding bash and GNU/Linux 14 # extensions not present on other systems. Practically, for now, it 15 # will use a subset that is present on GNU/Linux, macOS, *BSD, and 16 # illumos, documented here. 17 # - seq(1) 18 19 # The overall design is 20 # - a number of shell functions for various setup and micro tests 21 # - functions to wrap those in loops 22 # - functions to format output 23 # - written (for now) for nerds; errors understandable from reading 24 # the sources are good enough 25 26 # The sh style is 27 # quote when necessary 28 # avoid quotes when static analysis says that is safe 29 # use ${var} always 30 31 # TODO 32 # - figure out how to set limits on remote process 33 # - figure out how to set UNISON for remote process 34 # - loop over sizes 35 36 log () { 37 now=`date +%s` 38 echo "check-memory: $now $*" 39 } 40 41 fatal () { 42 log FATAL $* 43 exit 1 44 } 45 46 CONTAINER=/tmp 47 DIR=${CONTAINER}/UNISON-TEST 48 49 goto_dir () { 50 cd ${CONTAINER} 51 if [ \! -d ${DIR} ]; then 52 mkdir ${DIR} || fatal mkdir 53 fi 54 cd ${DIR} || fatal cd 55 56 # Ensure no other uid can access the socket. 57 chmod 700 . || fatal chmod 58 59 if [ -e local -o -e remote ]; then 60 fatal source or remote exists at startup 61 fi 62 63 # Avoid creating archive files in the user's directory. 64 # Ensure that tests start out without leftover state. 65 UNISON=${DIR}/.unison.local 66 export UNISON 67 if [ -e .unison ]; then 68 fatal .unison exists at startup 69 fi 70 } 71 72 # Clean up all state we created. 73 fini () { 74 # Be extra careful about removals. 75 if [ -d ../UNISON-TEST ]; then 76 rm -rf local remote .unison.local .unison.remote s 77 else 78 fatal fini not in UNISON-TEST 79 fi 80 } 81 82 # Create N*M small files. 83 init_N_M () { 84 if [ -e local ]; then 85 fatal init_N_m local exists 86 fi 87 mkdir local 88 89 for n in $(seq $1); do 90 mkdir local/$n 91 for m in $(seq $2); do 92 echo $n $m > local/$n/$m 93 done 94 done 95 } 96 97 touch_N_M_all () { 98 if [ ! -e local ]; then 99 fatal touch_N_m local does not exist 100 fi 101 102 find local -type f | while read f; do 103 date >> $f 104 done 105 } 106 107 # Set limit of arg1 to arg2. 108 # POSIX defines very little: 109 # https://pubs.opengroup.org/onlinepubs/9799919799/ 110 # and in particular does not define: 111 # -m -v 112 # 113 # POSIX defines setrlimit(2): 114 # https://pubs.opengroup.org/onlinepubs/9799919799/functions/getrlimit.html 115 # 116 # Generally, m (not POSIX) corresponds to RLIMIT_RSS (not POSIX) and v 117 # (not POSIX) corresponds to RLIMIT_AS (POSIX). 118 # 119 # With -v/RLIMIT_AS, address space, not memory usage, is limited, and 120 # thus there is a larger base load of VA surely not backed by pages. 121 # 122 # On older macOS, "d" and "m" do not seem to limit malloc. 123 # On NetBSD 10, "d" and "m" do not limit malloc. 124 # v limits address space, with background usage higher than one 125 # would guess. Also, v limits are not repeatable. 126 # On Debian 12, "d" limits malloc and "m" does not. 127 # 128 limit () { 129 flag=-"$1" 130 log limit flag $flag 131 old=`ulimit $flag` 132 ulimit $flag $2 133 new=`ulimit $flag` 134 135 log limit flag $flag old $old req $2 new $new 136 } 137 138 limit_display () { 139 log SOFT 140 ulimit -S -a 141 if false; then 142 log HARD 143 ulimit -H -a 144 fi 145 } 146 147 start_server () { 148 UNISON=${DIR}/.unison.remote 149 export UNISON 150 151 unison -socket s $* & 152 sleep 1 153 } 154 155 # Perform a sync 156 # expect: local and remote already set up 157 # results: exit status stored in STATUS 158 do_sync () { 159 unison -killserver -batch $* local socket://{${DIR}/s}//${DIR}/remote 160 STATUS=$? 161 } 162 163 # For no good reason, pick 10x1000 = 10^4 files. 164 # Use the same memory limit for local and remote. The search space is 165 # too large, and finding the level at which one breaks is, for now, 166 # good enough. 167 # \todo Expand to take args, so it can be used in a loop. 168 simple_test () { 169 N=10 170 M=1000 171 172 # NetBSD 10 amd64 10 1000: 1373 bad 1376 ok 173 # NetBSD 10 amd64 20 1000: 1376 ok 174 # NetBSD 10 amd64 40 1000: 1376 ok 175 memory=1376 176 # NetBSD 10 amd64 10 1000: 84 bad 88 ok 177 # NetBSD 10 amd64 20 1000: 124 bad 128 ok 178 # NetBSD 10 amd64 40 1000: 208 bad 212 ok 179 stack=88 180 181 # Create many files, N dirs of M files. 182 init_N_M ${N} ${M} 183 184 # Set limits. Set both d and m, because of surprising and not yet 185 # understood test results. 186 limit d ${memory} 187 limit m ${memory} 188 limit s ${stack} 189 limit_display 190 191 start_server -ignorearchives 192 do_sync -ignorearchives 193 # log a cryptic line, that can be grepped for and parsed programmatically 194 log "sync ${N} ${M} ${memory} ${stack} ${STATUS}" 195 196 touch_N_M_all 197 start_server 198 do_sync 199 log "sync-touch-all ${N} ${M} ${memory} ${stack} ${STATUS}" 200 } 201 202 # \todo Write a loop with binary search, to find the memory needed for a given test. 203 # \todo Write a loop over test sizes. 204 205 all () { 206 goto_dir 207 208 simple_test 209 210 fini 211 } 212 213 all