Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
P
presto
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Analytics
Analytics
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Commits
Open sidebar
SYNERG
presto
Commits
6170e62c
Commit
6170e62c
authored
Nov 08, 2020
by
RAHUL BHARDWAJ
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Partially completed IOPS/WSS analysis
parent
ef7374a8
Changes
6
Show whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
355 additions
and
26 deletions
+355
-26
code/cache.h
code/cache.h
+39
-1
code/exp2.cc
code/exp2.cc
+25
-14
code/experiments.cc
code/experiments.cc
+54
-1
code/plot-exp2.py
code/plot-exp2.py
+232
-9
code/run
code/run
+3
-0
code/util.h
code/util.h
+2
-1
No files found.
code/cache.h
View file @
6170e62c
...
@@ -61,6 +61,8 @@ public:
...
@@ -61,6 +61,8 @@ public:
long
evct_count
[
NUM_POOLS
][
NUM_HASHMAPS
];
// pools X hashmaps
long
evct_count
[
NUM_POOLS
][
NUM_HASHMAPS
];
// pools X hashmaps
long
evct_count_prewarmed
;
// pools X hashmaps
long
evct_count_prewarmed
;
// pools X hashmaps
long
*
io_count_vdisk
;
// vdisk
long
*
io_count_vdisk
;
// vdisk
double
*
avg_iops
;
double
*
wss_before_failure
;
bool
prewarm_till_first_eviction
;
// stop prewarming once a prewarm object get evicted
bool
prewarm_till_first_eviction
;
// stop prewarming once a prewarm object get evicted
bool
prewarm_at_head
;
// insert prewrming objects at front
bool
prewarm_at_head
;
// insert prewrming objects at front
...
@@ -102,9 +104,12 @@ public:
...
@@ -102,9 +104,12 @@ public:
this
->
singlepool
=
false
;
this
->
singlepool
=
false
;
this
->
evct_count_prewarmed
=
0
;
this
->
evct_count_prewarmed
=
0
;
this
->
prewarm_till_first_eviction
=
false
;
this
->
prewarm_till_first_eviction
=
false
;
this
->
prewarm_at_head
=
fals
e
;
this
->
prewarm_at_head
=
tru
e
;
this
->
io_count_vdisk
=
new
long
[
num_vdisks
];
this
->
io_count_vdisk
=
new
long
[
num_vdisks
];
this
->
avg_iops
=
new
double
[
num_vdisks
];
this
->
wss_before_failure
=
new
double
[
num_vdisks
];
memset
(
this
->
io_count_vdisk
,
0
,
num_vdisks
*
sizeof
(
long
));
memset
(
this
->
io_count_vdisk
,
0
,
num_vdisks
*
sizeof
(
long
));
for
(
i
=
0
;
i
<
NUM_HASHMAPS
;
i
++
)
for
(
i
=
0
;
i
<
NUM_HASHMAPS
;
i
++
)
{
{
...
@@ -150,6 +155,39 @@ public:
...
@@ -150,6 +155,39 @@ public:
ba3
=
ba1
[
i
]
=
new
BitArray
(
1UL
<<
HASH_CHARS_FOR_EGROUP
*
4
);
ba3
=
ba1
[
i
]
=
new
BitArray
(
1UL
<<
HASH_CHARS_FOR_EGROUP
*
4
);
oba3
=
oba1
[
i
]
=
new
BitArray
(
1UL
<<
HASH_CHARS_FOR_EGROUP
*
4
);
oba3
=
oba1
[
i
]
=
new
BitArray
(
1UL
<<
HASH_CHARS_FOR_EGROUP
*
4
);
}
}
const
char
*
tf
=
wload
->
tracefile
;
FILE
*
fp
=
fopen
(
tf
,
"r"
);
assert
(
fp
!=
nullptr
);
char
buf
[
MAX_REQ_LENGTH
],
*
word
,
*
saveptr
;
double
total_time
=
0
;
while
(
fgets
(
buf
,
MAX_REQ_LENGTH
,
fp
))
{
word
=
strtok_r
(
buf
,
","
,
&
saveptr
);
//req->type = word[0];
word
=
strtok_r
(
nullptr
,
","
,
&
saveptr
);
int
disk_id
=
-
1
;
for
(
int
j
=
0
;
j
<
num_vdisks
;
j
++
)
{
if
(
wload
->
vdisks
[
j
].
vdisk_id
==
strtol
(
word
,
nullptr
,
0
))
disk_id
=
j
;
}
avg_iops
[
disk_id
]
+=
1.0
;
word
=
strtok_r
(
nullptr
,
","
,
&
saveptr
);;
word
=
strtok_r
(
nullptr
,
","
,
&
saveptr
);
word
=
strtok_r
(
nullptr
,
","
,
&
saveptr
);
total_time
=
strtod
(
word
,
nullptr
);
}
for
(
i
=
0
;
i
<
num_vdisks
;
i
++
)
{
avg_iops
[
i
]
/=
total_time
;
}
fclose
(
fp
);
//else if (type_bitmap == BITSET)
//else if (type_bitmap == BITSET)
//{
//{
// bs1 = new BitSet *[num_vdisks];
// bs1 = new BitSet *[num_vdisks];
...
...
code/exp2.cc
View file @
6170e62c
...
@@ -56,16 +56,24 @@ void exp2()
...
@@ -56,16 +56,24 @@ void exp2()
// {5, "5%"},
// {5, "5%"},
// {10, "10%"},
// {10, "10%"},
// {25, "25%"},
// {25, "25%"},
// {50, "50%"}
};
// {50, "50%"}
,
{
100
,
"100
%"
}};
{
75
,
"75
%"
}};
// {100, "100%"}};
// {100, "100%"}};
// struct ns_pair prewarm_rate_limit[] = {
// // // {0, "0MBps"},
// {20, "20MBps"},
// {50, "50MBps"},
// {100, "100MBps"}
// {500, "500MBps"},
// {10000, "10000MBps"}};
struct
ns_pair
prewarm_rate_limit
[]
=
{
struct
ns_pair
prewarm_rate_limit
[]
=
{
// {
0, "0
MBps"},
// {
15, "15
MBps"},
{
20
,
"2
0MBps"
},
// {30, "3
0MBps"},
{
50
,
"50MBps"
},
{
100
,
"100MBps"
}};
{
100
,
"100MBps"
}
// {200, "200MBps"},
{
500
,
"5
00MBps"
}};
// {1000, "10
00MBps"}};
{
10000
,
"10
000MBps"
}};
// {60000, "6
000MBps"}};
// {500, "500MBps"}};
// {500, "500MBps"}};
struct
ns_pair
heuristic
[]
=
{
struct
ns_pair
heuristic
[]
=
{
...
@@ -105,11 +113,13 @@ void exp2()
...
@@ -105,11 +113,13 @@ void exp2()
rp
->
cache
=
cache
;
rp
->
cache
=
cache
;
rp
->
sync_reqs
=
true
;
rp
->
sync_reqs
=
true
;
rp
->
pcache
=
pcache
;
rp
->
pcache
=
pcache
;
rp
->
part_scheme_hm1
=
HM1_NO_PART
;
// rp->part_scheme_hm1 = HM1_NO_PART;
rp
->
part_scheme_hm1
=
HM1_WSS_SHARE
;
rp
->
part_scheme_hm1_hm3
=
HM1_HM3_NO_PART
;
rp
->
part_scheme_hm1_hm3
=
HM1_HM3_NO_PART
;
rp
->
trace_file
=
wload
->
tracefile
;
rp
->
trace_file
=
wload
->
tracefile
;
rp
->
wss_window
=
600
;
rp
->
wss_window
=
600
;
rp
->
wss_stride
=
300
;
rp
->
wss_stride
=
300
;
// rp->moment = 2400;
rp
->
moment
=
5400
;
rp
->
moment
=
5400
;
asprintf
(
&
basedirname
,
PRESTO_BASEPATH
"results/%s/%s/"
,
"exp2"
,
wload
->
name
);
asprintf
(
&
basedirname
,
PRESTO_BASEPATH
"results/%s/%s/"
,
"exp2"
,
wload
->
name
);
...
@@ -120,7 +130,7 @@ void exp2()
...
@@ -120,7 +130,7 @@ void exp2()
rq
=
new
struct
sync_rq
;
rq
=
new
struct
sync_rq
;
rq
->
tfname
=
rp
->
trace_file
;
rq
->
tfname
=
rp
->
trace_file
;
rp
->
rq
=
rq
;
rp
->
rq
=
rq
;
pthread_create
(
&
io_thread
,
nullptr
,
get_req_from_vdisk_sync
,
(
void
*
)
rq
);
pthread_create
(
&
io_thread
,
nullptr
,
get_req_from_vdisk_sync
,
rq
);
pthread_join
(
io_thread
,
nullptr
);
pthread_join
(
io_thread
,
nullptr
);
}
}
for
(
ssr
=
0
;
ssr
<
LEN
(
snapshot_rate
,
ns_pair
);
ssr
++
)
// snapshot rate
for
(
ssr
=
0
;
ssr
<
LEN
(
snapshot_rate
,
ns_pair
);
ssr
++
)
// snapshot rate
...
@@ -294,7 +304,7 @@ void *run_exp2(void *data)
...
@@ -294,7 +304,7 @@ void *run_exp2(void *data)
res_wss
[
i
]
=
new
long
[
MAX_TIME_SEC
+
2
-
wss_stride
];
res_wss
[
i
]
=
new
long
[
MAX_TIME_SEC
+
2
-
wss_stride
];
}
}
res_usage
=
new
long
[
MAX_TIME_SEC
+
1
];
res_usage
=
new
long
[
MAX_TIME_SEC
+
1
];
prewarmed_evct_cnt
=
new
long
[
MAX_TIME_SEC
+
1
];
prewarmed_evct_cnt
=
new
long
[
MAX_TIME_SEC
+
1
];
res_evct
=
new
long
*
[
NUM_POOLS
];
res_evct
=
new
long
*
[
NUM_POOLS
];
res_evct
[
POOL_SINGLE
]
=
new
long
[
MAX_TIME_SEC
+
1
];
res_evct
[
POOL_SINGLE
]
=
new
long
[
MAX_TIME_SEC
+
1
];
res_evct
[
POOL_MULTI
]
=
new
long
[
MAX_TIME_SEC
+
1
];
res_evct
[
POOL_MULTI
]
=
new
long
[
MAX_TIME_SEC
+
1
];
...
@@ -406,6 +416,7 @@ void *run_exp2(void *data)
...
@@ -406,6 +416,7 @@ void *run_exp2(void *data)
if
(
request
->
ts
>=
(
wss_stride_epoch
+
1
)
*
wss_stride
)
if
(
request
->
ts
>=
(
wss_stride_epoch
+
1
)
*
wss_stride
)
{
{
printf
(
"%d, %f
\n
"
,
wss_stride_epoch
,
request
->
ts
);
if
(
request
->
ts
>=
wss_window
)
if
(
request
->
ts
>=
wss_window
)
{
{
wsses
=
cache
->
estimate_wss
(
wss_window
,
true
);
wsses
=
cache
->
estimate_wss
(
wss_window
,
true
);
...
@@ -453,7 +464,7 @@ void *run_exp2(void *data)
...
@@ -453,7 +464,7 @@ void *run_exp2(void *data)
max_epochs
=
epoch
+
1
;
max_epochs
=
epoch
+
1
;
max_plot_epochs
=
plot_epoch
+
1
;
max_plot_epochs
=
plot_epoch
+
1
;
max_wss_stride_epochs
=
wss_stride_epoch
+
1
;
max_wss_stride_epochs
=
wss_stride_epoch
+
1
;
printf
(
"%d
\n
"
,
max_wss_stride_epochs
);
for
(
i
=
0
;
i
<
num_vdisks
;
i
++
)
for
(
i
=
0
;
i
<
num_vdisks
;
i
++
)
{
{
snprintf
(
hr_fname
,
MAX_FILENAME_LEN
,
"%sCOLD-VDISK-%d-HR.csv"
,
base_fname
,
cache
->
vdisk_ids
[
i
]);
snprintf
(
hr_fname
,
MAX_FILENAME_LEN
,
"%sCOLD-VDISK-%d-HR.csv"
,
base_fname
,
cache
->
vdisk_ids
[
i
]);
...
@@ -474,7 +485,7 @@ void *run_exp2(void *data)
...
@@ -474,7 +485,7 @@ void *run_exp2(void *data)
}
}
for
(
j
=
0
;
j
<
max_wss_stride_epochs
;
j
++
)
for
(
j
=
0
;
j
<
max_wss_stride_epochs
;
j
++
)
fprintf
(
wss_file
,
"%d,%ld
\n
"
,
j
,
res_wss
[
i
][
j
]);
fprintf
(
wss_file
,
"%d,%ld
\n
"
,
j
,
res_wss
[
i
][
j
]);
cache
->
wss_before_failure
[
i
]
=
res_wss
[
i
][
rp
->
moment
/
rp
->
wss_stride
-
1
];
fflush
(
hr_file
);
fflush
(
hr_file
);
fclose
(
hr_file
);
fclose
(
hr_file
);
fflush
(
obj_file
);
fflush
(
obj_file
);
...
@@ -598,7 +609,7 @@ prewarm_step:
...
@@ -598,7 +609,7 @@ prewarm_step:
}
}
while
(
request
->
ts
>=
counter
&&
(
rem_vdisks_prewarm
>
0
||
pre_size_limit
>=
SIZE_HM1_OBJ
))
while
(
request
->
ts
>=
counter
&&
(
rem_vdisks_prewarm
>
0
||
pre_size_limit
>=
SIZE_HM1_OBJ
))
{
{
if
(
cache
->
prewarm_till_first_eviction
&&
cache
->
evct_count_prewarmed
>
0
)
if
(
cache
->
prewarm_till_first_eviction
&&
cache
->
evct_count_prewarmed
>
0
)
break
;
break
;
prewarm_set_used_total
=
0
;
prewarm_set_used_total
=
0
;
for
(
i
=
0
;
i
<
num_vdisks
;
i
++
)
for
(
i
=
0
;
i
<
num_vdisks
;
i
++
)
...
...
code/experiments.cc
View file @
6170e62c
...
@@ -452,7 +452,24 @@ BitArray **analyze_snapshots(Cache *cache, int max_epochs, int ssr, long pre_siz
...
@@ -452,7 +452,24 @@ BitArray **analyze_snapshots(Cache *cache, int max_epochs, int ssr, long pre_siz
mem_limits
[
i
]
=
((
double
)
cache
->
ba1
[
i
]
->
onbits
/
total_objs
)
*
(
pre_size_limit
/
2.0
);
mem_limits
[
i
]
=
((
double
)
cache
->
ba1
[
i
]
->
onbits
/
total_objs
)
*
(
pre_size_limit
/
2.0
);
mem_limits
[
i
]
=
pre_size_limit
/
2.0
;
mem_limits
[
i
]
=
pre_size_limit
/
2.0
;
}
}
else
if
(
part_scheme_hm1
==
HM1_WSS_SHARE
)
{
printf
(
"Using wss share
\n
"
);
double
weight
[
num_vdisks
];
double
total_weight
=
0
;
for
(
i
=
0
;
i
<
num_vdisks
;
i
++
)
{
weight
[
i
]
=
cache
->
avg_iops
[
i
]
/
cache
->
wss_before_failure
[
i
];
// printf("disk %d, %f, %f\n", i, cache->avg_iops[i], cache->wss_before_failure[i]);
total_weight
+=
weight
[
i
];
}
for
(
i
=
0
;
i
<
num_vdisks
;
i
++
)
{
printf
(
"disk %d, %f, %f, %f
\n
"
,
i
,
weight
[
i
]
/
total_weight
,
cache
->
avg_iops
[
i
],
cache
->
wss_before_failure
[
i
]);
mem_limits
[
i
]
=
(
pre_size_limit
*
weight
[
i
])
/
(
2
*
total_weight
);
}
mem_limits
[
i
]
=
pre_size_limit
/
2.0
;
}
rem_mem
=
rem_vdisks
=
0
;
rem_mem
=
rem_vdisks
=
0
;
memset
(
obj_pos
,
0
,
sizeof
(
int
)
*
(
num_vdisks
+
1
));
memset
(
obj_pos
,
0
,
sizeof
(
int
)
*
(
num_vdisks
+
1
));
for
(
i
=
0
;
i
<
num_vdisks
;
i
++
)
for
(
i
=
0
;
i
<
num_vdisks
;
i
++
)
...
@@ -468,6 +485,42 @@ BitArray **analyze_snapshots(Cache *cache, int max_epochs, int ssr, long pre_siz
...
@@ -468,6 +485,42 @@ BitArray **analyze_snapshots(Cache *cache, int max_epochs, int ssr, long pre_siz
}
}
analyze_scores
(
agg
[
i
],
scores
[
i
],
&
obj_pos
[
i
],
SIZE_HM3_OBJ
,
mem_limits
[
i
]);
analyze_scores
(
agg
[
i
],
scores
[
i
],
&
obj_pos
[
i
],
SIZE_HM3_OBJ
,
mem_limits
[
i
]);
while
(
part_scheme_hm1
==
HM1_WSS_SHARE
&&
rem_mem
>
SIZE_HM1_OBJ
)
{
//fprintf(stderr, "rem_mem: %lu, rem_vdisks: %lu\n", rem_mem, rem_vdisks);
if
(
rem_mem
>=
rem_vdisks
*
SIZE_HM1_OBJ
)
{
rem_mem_old
=
rem_mem
;
rem_vdisks_old
=
rem_vdisks
;
rem_mem
=
rem_vdisks
=
0
;
for
(
i
=
0
;
i
<
num_vdisks
;
i
++
)
{
if
(
mem_limits
[
i
]
==
0
)
continue
;
mem_limits
[
i
]
=
(
double
)
rem_mem_old
/
rem_vdisks_old
;
k
=
analyze_scores
(
agg
[
i
],
scores
[
i
],
&
obj_pos
[
i
],
SIZE_HM1_OBJ
,
mem_limits
[
i
]);
if
(
k
<
SIZE_HM1_OBJ
)
rem_vdisks
++
;
else
{
mem_limits
[
i
]
=
0
;
rem_mem
+=
k
;
}
}
}
else
{
for
(
i
=
0
;
i
<
num_vdisks
&&
rem_mem
>=
SIZE_HM1_OBJ
;
i
++
)
{
if
(
mem_limits
[
i
]
==
0
)
continue
;
analyze_scores
(
agg
[
i
],
scores
[
i
],
&
obj_pos
[
i
],
SIZE_HM1_OBJ
,
SIZE_HM1_OBJ
);
rem_mem
-=
SIZE_HM1_OBJ
;
}
}
}
while
(
part_scheme_hm1
==
HM1_FAIR_SHARE
&&
rem_mem
>
SIZE_HM1_OBJ
)
while
(
part_scheme_hm1
==
HM1_FAIR_SHARE
&&
rem_mem
>
SIZE_HM1_OBJ
)
{
{
//fprintf(stderr, "rem_mem: %lu, rem_vdisks: %lu\n", rem_mem, rem_vdisks);
//fprintf(stderr, "rem_mem: %lu, rem_vdisks: %lu\n", rem_mem, rem_vdisks);
...
...
code/plot-exp2.py
View file @
6170e62c
...
@@ -24,14 +24,16 @@ def main(wload, target_cache_size):
...
@@ -24,14 +24,16 @@ def main(wload, target_cache_size):
# scores = ['100%', '50%', '33%', '25%', '20%', '1', '2', '3', '4', '5', '1%', '5%', '10%', '15%', '20%']
# scores = ['100%', '50%', '33%', '25%', '20%', '1', '2', '3', '4', '5', '1%', '5%', '10%', '15%', '20%']
# scname = ['100\%', '50\%', '33\%', '25\%', '20\%', '1', '2', '3', '4', '5', '1\%', '5\%', '10\%', '15\%', '20\%']
# scname = ['100\%', '50\%', '33\%', '25\%', '20\%', '1', '2', '3', '4', '5', '1\%', '5\%', '10\%', '15\%', '20\%']
# mlimits = ['5%', '10%', '25%', '50%', '75%', '100%']
# mlimits = ['5%','10%', '25%', '50%', '75%']
# mname = ['5%', '10%', '25%', '50%', '75%', '100%']
# mname = ['5%', '10%', '25%', '50%', '75%']
# mlimits = ['10%', '25%', '50%', '75%', '100%']
# mname = ['10%', '25%', '50%', '75%','100%']
mlimits
=
[
'75
%
'
]
mname
=
[
'75
%
'
]
# mlimits = ['100%']
# mname = ['100%']
# mlimits = ['75%']
# mlimits = ['75%']
# mname = ['75%']
# mname = ['75%']
# mlimits = ['50%']
# mname = ['50%']
mlimits
=
[
'100
%
'
]
mname
=
[
'100
%
'
]
# mlimits = ['75%', '100%']
# mlimits = ['75%', '100%']
# mname = ['75%', '100%']
# mname = ['75%', '100%']
...
@@ -41,9 +43,18 @@ def main(wload, target_cache_size):
...
@@ -41,9 +43,18 @@ def main(wload, target_cache_size):
# pratenames = ['∞', '500 MBps', '100 MBps', '50 MBps']
# pratenames = ['∞', '500 MBps', '100 MBps', '50 MBps']
# prewarmrates = ['1000MBps']
# prewarmrates = ['1000MBps']
# pratenames = ['1000 MBps']
# pratenames = ['1000 MBps']
prewarmrates
=
[
'100
00
MBps'
]
prewarmrates
=
[
'100MBps'
]
pratenames
=
[
'100
00
MBps'
]
pratenames
=
[
'100 MBps'
]
# prewarmrates = ['0MBps','15MBps', '30MBps', '100MBps', '500MBps', '1000MBps']
# pratenames = ['0MBps','15 MBps', '30 MBps', '100 MBps', '500 MBps', '10000 MBps']
# prewarmrates = ['50MBps', '100MBps', '200MBps', '500MBps', '1000MBps']
# pratenames = ['50 MBps', '100 MBps', '200 MBps', '500 MBps', '10000 MBps']
varying_prewarm_rate_data
=
[]
# moment = 2400
moment
=
5400
moment
=
5400
wss_window
=
600
wss_window
=
600
wss_stride
=
300
wss_stride
=
300
...
@@ -54,6 +65,10 @@ def main(wload, target_cache_size):
...
@@ -54,6 +65,10 @@ def main(wload, target_cache_size):
wload_file
=
'traces/'
+
wload
+
'.csv'
wload_file
=
'traces/'
+
wload
+
'.csv'
with
open
(
wload_file
)
as
wfile
:
with
open
(
wload_file
)
as
wfile
:
vdisks
=
list
(
csv
.
reader
(
wfile
))
vdisks
=
list
(
csv
.
reader
(
wfile
))
# print(len(vdisks))
varying_prewarm_rate_data
=
[[]
for
x
in
range
(
len
(
vdisks
))]
varying_prewarm_size_data
=
[[]
for
x
in
range
(
len
(
vdisks
))]
##################################################################################################################
##################################################################################################################
######## NOFAIL
######## NOFAIL
...
@@ -81,7 +96,9 @@ def main(wload, target_cache_size):
...
@@ -81,7 +96,9 @@ def main(wload, target_cache_size):
with
open
(
basedir_nop
+
'COLD-VDISK-'
+
vdisks
[
vd
][
0
]
+
'-OBJ.csv'
)
as
nopfile
:
with
open
(
basedir_nop
+
'COLD-VDISK-'
+
vdisks
[
vd
][
0
]
+
'-OBJ.csv'
)
as
nopfile
:
lines
=
list
(
csv
.
reader
(
nopfile
))
lines
=
list
(
csv
.
reader
(
nopfile
))
# print(len(lines), len(wof_misses))
for
ln
in
range
(
len
(
lines
)):
for
ln
in
range
(
len
(
lines
)):
# print(lines[ln], len(wof_misses))
wof_misses
[
ln
]
+=
int
(
lines
[
ln
][
4
])
+
int
(
lines
[
ln
][
5
])
wof_misses
[
ln
]
+=
int
(
lines
[
ln
][
4
])
+
int
(
lines
[
ln
][
5
])
##################################################################################################################
##################################################################################################################
...
@@ -133,6 +150,7 @@ def main(wload, target_cache_size):
...
@@ -133,6 +150,7 @@ def main(wload, target_cache_size):
lines
=
list
(
csv
.
reader
(
nopfile
))
lines
=
list
(
csv
.
reader
(
nopfile
))
wss_step
=
int
(
wss_window
/
wss_stride
)
wss_step
=
int
(
wss_window
/
wss_stride
)
wss_epoch
=
int
((
moment
/
60
)
/
(
wss_window
/
60
))
*
wss_step
-
1
wss_epoch
=
int
((
moment
/
60
)
/
(
wss_window
/
60
))
*
wss_step
-
1
print
(
len
(
lines
),
wss_epoch
,
vd
)
wss_before_failure_vdisk
.
append
(
round
(
int
(
lines
[
wss_epoch
][
1
])
/
1048576
,
2
))
wss_before_failure_vdisk
.
append
(
round
(
int
(
lines
[
wss_epoch
][
1
])
/
1048576
,
2
))
with
open
(
basedir_nop
+
'COLD-VDISK-'
+
vdisks
[
vd
][
0
]
+
'-OBJ.csv'
)
as
nopfile
:
with
open
(
basedir_nop
+
'COLD-VDISK-'
+
vdisks
[
vd
][
0
]
+
'-OBJ.csv'
)
as
nopfile
:
...
@@ -171,6 +189,7 @@ def main(wload, target_cache_size):
...
@@ -171,6 +189,7 @@ def main(wload, target_cache_size):
total_evictions
.
append
(
float
(
line
[
1
])
+
float
(
line
[
2
]))
total_evictions
.
append
(
float
(
line
[
1
])
+
float
(
line
[
2
]))
prewarmed_objects_evictions
.
append
(
float
(
line
[
3
]))
prewarmed_objects_evictions
.
append
(
float
(
line
[
3
]))
evictions_due_to_load
.
append
(
float
(
line
[
1
])
+
float
(
line
[
2
])
-
float
(
line
[
3
]))
evictions_due_to_load
.
append
(
float
(
line
[
1
])
+
float
(
line
[
2
])
-
float
(
line
[
3
]))
print
(
basedir_pre
)
with
open
(
basedir_pre
+
'ALL-HR.csv'
)
as
prefile
:
with
open
(
basedir_pre
+
'ALL-HR.csv'
)
as
prefile
:
lines
=
list
(
csv
.
reader
(
prefile
))
lines
=
list
(
csv
.
reader
(
prefile
))
for
line
in
lines
:
for
line
in
lines
:
...
@@ -213,6 +232,8 @@ def main(wload, target_cache_size):
...
@@ -213,6 +232,8 @@ def main(wload, target_cache_size):
lines
=
list
(
csv
.
reader
(
prefile
))
lines
=
list
(
csv
.
reader
(
prefile
))
for
line
in
lines
:
for
line
in
lines
:
hr_vdisk
.
append
(
float
(
line
[
1
]))
hr_vdisk
.
append
(
float
(
line
[
1
]))
varying_prewarm_rate_data
[
vd
]
.
append
(
sum
(
hr_vdisk
[
moment
:
moment
+
5
*
60
])
/
300
)
varying_prewarm_size_data
[
vd
]
.
append
(
sum
(
hr_vdisk
[
moment
:
moment
+
5
*
60
])
/
300
)
hrs_wp_vdisk
.
append
(
hr_vdisk
)
hrs_wp_vdisk
.
append
(
hr_vdisk
)
pre_count
=
0
pre_count
=
0
...
@@ -513,6 +534,207 @@ def main(wload, target_cache_size):
...
@@ -513,6 +534,207 @@ def main(wload, target_cache_size):
##################################################################################################################
##################################################################################################################
######## EXTRAS
######## EXTRAS
##################################################################################################################
##################################################################################################################
colors
=
[
'tab:blue'
,
'tab:orange'
,
'tab:green'
,
'tab:purple'
,
'tab:cyan'
,
'tab:olive'
,
'tab:brown'
,
'tab:pink'
]
prewarmrates
=
[
0
,
15
,
30
,
100
,
200
,
1000
]
prewarmsize
=
[
0
,
5
,
10
,
25
,
50
,
75
]
# prewarmrates = [15,30,100,200,1000]
# prewarmsize = [5,10,25,50,75]
# prewarmrates = [0, 50,100,200,500, 1000]
# prewarmsize = [0,10,25,50,75,100]
# prewarmrates = [50,100,200,500, 1000]
# prewarmsize = [10,25,50,75,100]
prate_index
=
{}
psize_index
=
{}
for
i
in
range
(
len
(
prewarmrates
)):
prate_index
[
prewarmrates
[
i
]]
=
i
for
i
in
range
(
len
(
prewarmsize
)):
psize_index
[
prewarmsize
[
i
]]
=
i
varying_prewarm_rate_per_vdisk
=
np
.
zeros
((
len
(
vdisks
),
len
(
prewarmrates
),
len
(
prewarmsize
))
)
for
ssr
in
range
(
len
(
snaprates
)):
for
mlim
in
range
(
len
(
prewarmsize
)):
for
prate
in
range
(
len
(
prewarmrates
)):
for
hrstc
in
range
(
len
(
heuristics
)):
for
ffn
in
range
(
len
(
frerern_func
)
if
hrstc
==
2
else
1
):
basedir_nop
=
'results/exp2/'
+
wload
+
'/'
+
snaprates
[
ssr
]
+
'/'
basedir_pre
=
basedir_nop
+
str
(
prewarmsize
[
mlim
])
+
'
%-
'
+
str
(
prewarmrates
[
prate
])
+
'MBps-'
+
\
heuristics
[
hrstc
]
+
'-'
+
((
frerern_func
[
ffn
]
+
'-'
)
if
(
hrstc
==
2
)
else
''
)
+
\
'x/'
for
vd
in
range
(
len
(
vdisks
)):
hr_vdisk
=
[]
if
prate
==
0
or
mlim
==
0
:
with
open
(
basedir_nop
+
'COLD-VDISK-'
+
vdisks
[
vd
][
0
]
+
'-HR.csv'
)
as
prefile
:
lines
=
list
(
csv
.
reader
(
prefile
))
else
:
with
open
(
basedir_pre
+
'VDISK-'
+
vdisks
[
vd
][
0
]
+
'-HR.csv'
)
as
prefile
:
lines
=
list
(
csv
.
reader
(
prefile
))
for
line
in
lines
:
hr_vdisk
.
append
(
float
(
line
[
1
]))
varying_prewarm_rate_data
[
vd
]
.
append
(
sum
(
hr_vdisk
[
moment
:
moment
+
5
*
60
])
/
300
)
varying_prewarm_size_data
[
vd
]
.
append
(
sum
(
hr_vdisk
[
moment
:
moment
+
5
*
60
])
/
300
)
# print(varying_prewarm_rate_da)
varying_prewarm_rate_per_vdisk
[
vd
][
prate
][
mlim
]
=
sum
(
hr_vdisk
[
moment
:
moment
+
5
*
60
])
/
300
# Plot of 5-min hit ratio average by varying prewarm rates
color_list
=
[
'tab:blue'
,
'tab:orange'
,
'tab:green'
,
'tab:purple'
,
'tab:cyan'
,
'tab:olive'
]
# plt.clf()
# directory = basedir_nop
# for vd in range(len(vdisks)):
# plt.plot(prewarmrates,varying_prewarm_rate_data[vd],
# color = colors[vd], linewidth=1.2, label = 'vDisk' + str(vd+1))
# plt.legend(bbox_to_anchor=(1.03,1.0), loc= 'upper left', fancybox=True,prop={'size': 10})
# plt.xlabel('Prewarm Rates')
# plt.ylabel('5-min average HR')
# plt.ylim(0.4,1);
# plt.savefig(directory+mlimits[0]+'-'+'variable', dpi=300, bbox_inches='tight')
#Disk wise plots
# print(varying_prewarm_rate_per_vdisk[1][1][3])
# plt.clf()
# directory = basedir_nop
# for vd in range(len(vdisks)):
# plt.clf()
# for i in range(len(prewarmsize)):
# if vd==0:
# print(varying_prewarm_rate_per_vdisk[vd,:,i], prewarmsize[i])
# plt.plot(prewarmrates,varying_prewarm_rate_per_vdisk[vd,:,i],
# color = color_list[i], linewidth=1.2, label = 'Prewarm size = ' + str(prewarmsize[i]))
# plt.xlabel('Prewarm Rates')
# plt.ylabel('5-min average HR')
# plt.legend(bbox_to_anchor=(1.03,1.0), loc= 'upper left', fancybox=True,prop={'size': 10})
# plt.ylim(0,1);
# plt.savefig(directory+'diskWiseData/'+'variable-rate-vdisk'+str(vd), dpi=300, bbox_inches='tight')
############################################################################################
# Plot of 5-min hit ratio average by varying prewarm size
# plt.clf()
# directory = basedir_nop
# prewarmsize = [5,10,25,50,75]
# for vd in range(len(vdisks)):
# # print(varying_prewarm_size_data[vd])
# plt.plot(prewarmsize,varying_prewarm_size_data[vd],
# color = colors[vd], linewidth=1.2, label = 'vDisk' + str(vd+1))
# plt.legend(bbox_to_anchor=(1.03,1.0), loc= 'upper left', fancybox=True,prop={'size': 10})
# plt.xlabel('Prewarm Sizes')
# plt.ylabel('5-min average HR')
# # plt.xlim(0,6000)
# plt.ylim(0.4,1);
# # plt.xscale('log')
# plt.savefig(directory + "variable-" + prewarmrates[0], dpi=300, bbox_inches='tight')
# print(directory)
#Disk wise plots
# plt.clf()
# directory = basedir_nop
# for vd in range(len(vdisks)):
# plt.clf()
# for i in range(len(prewarmrates)):
# if vd==0:
# print(varying_prewarm_rate_per_vdisk[vd,i,:], prewarmrates[i])
# plt.plot(prewarmsize,varying_prewarm_rate_per_vdisk[vd,i,:],
# color = color_list[i], linewidth=1.2, label = 'Prewarm rate = ' + str(prewarmrates[i]))
# plt.xlabel('Prewarm Size')
# plt.ylabel('5-min average HR')
# plt.legend(bbox_to_anchor=(1.03,1.0), loc= 'upper left', fancybox=True,prop={'size': 10})
# plt.ylim(0,1);
# plt.savefig(directory+'diskWiseData/'+'variable-size-vdisk'+str(vd), dpi=300, bbox_inches='tight')
# Plot of slopes vs wss
plt
.
clf
()
directory
=
basedir_nop
x
=
wss_before_failure_vdisk
for
i
in
range
(
len
(
prewarmrates
)):
plt
.
clf
()
# slope = np.zeros(len(vdisks))
slope
=
{}
for
vd
in
range
(
len
(
vdisks
)):
if
(
wss_before_failure_vdisk
[
vd
]
not
in
slope
.
keys
()):
slope
[
wss_before_failure_vdisk
[
vd
]]
=
(
varying_prewarm_rate_per_vdisk
[
vd
][
i
][
5
]
-
varying_prewarm_rate_per_vdisk
[
vd
][
i
][
1
])
/
wss_before_failure_vdisk
[
vd
]
x
.
sort
()
y
=
[
slope
[
l
]
for
l
in
x
]
# print(x,y)
# print(wss_before_failure_vdisk,slope)
plt
.
plot
(
x
,
y
,
color
=
color_list
[
i
],
linewidth
=
1.2
,
label
=
'Rate = '
+
str
(
prewarmrates
[
i
]))
x1
=
np
.
linspace
(
x
[
0
],
x
[
len
(
x
)
-
1
],
100
)
y1
=
y
[
0
]
*
x
[
0
]
/
x1
# print(x1,y1)
plt
.
plot
(
x1
,
y1
,
color
=
color_list
[(
i
+
1
)
%
len
(
color_list
)],
linestyle
=
'dashed'
,
label
=
'K/x'
)
y1
=
y
[
0
]
*
(
x
[
0
]
**
2
)
/
(
x1
**
2
)
plt
.
plot
(
x1
,
y1
,
color
=
color_list
[(
i
+
2
)
%
len
(
color_list
)],
linestyle
=
'dashed'
,
label
=
'K/x^2'
)
plt
.
xlabel
(
'WSS of the disks'
)
plt
.
ylabel
(
'Benefit from increasing prewarm set size'
)
plt
.
legend
(
bbox_to_anchor
=
(
1.01
,
1.0
),
loc
=
'upper left'
,
fancybox
=
True
,
prop
=
{
'size'
:
10
})
plt
.
savefig
(
directory
+
'slope/'
+
'slope-versus-wss-'
+
str
(
prewarmrates
[
i
])
+
'MBps'
,
dpi
=
300
,
bbox_inches
=
'tight'
)
############################################################################################
colors
=
[
'tab:blue'
,
'tab:orange'
,
'tab:green'
,
'tab:purple'
,
'tab:cyan'
,
'tab:olive'
,
'tab:brown'
,
'tab:pink'
]
classes
=
[]
for
vd
in
range
(
len
(
vdisks
)):
classes
.
append
(
'vDisk '
+
str
(
vd
+
1
))
directory
=
basedir_nop
for
i
in
range
(
len
(
prewarmrates
)):
for
j
in
range
(
len
(
prewarmsize
)):
plt
.
clf
()
for
vd
in
range
(
len
(
vdisks
)):
plt
.
scatter
(
wss_before_failure_vdisk
[
vd
],
varying_prewarm_rate_per_vdisk
[
vd
][
i
][
j
],
c
=
colors
[
vd
],
label
=
classes
[
vd
],
zorder
=
2
+
vd
)
plt
.
xlabel
(
'WSS before failure (MB)'
)
plt
.
ylabel
(
'5 min average HR'
)
plt
.
legend
(
bbox_to_anchor
=
(
1.01
,
1.0
),
loc
=
'upper left'
,
fancybox
=
True
,
prop
=
{
'size'
:
10
})
plt
.
savefig
(
directory
+
'WssVariation/'
+
str
(
prewarmsize
[
j
])
+
'
%-
'
+
str
(
prewarmrates
[
i
])
+
'MBps'
,
dpi
=
300
,
bbox_inches
=
'tight'
)
############################################################################################
colors
=
[
'tab:blue'
,
'tab:orange'
,
'tab:green'
,
'tab:purple'
,
'tab:cyan'
,
'tab:olive'
,
'tab:brown'
,
'tab:pink'
]
classes
=
[]
for
vd
in
range
(
len
(
vdisks
)):
classes
.
append
(
'vDisk '
+
str
(
vd
+
1
))
iops
=
[]
if
wload
==
'wload_6gb'
:
iops
=
[
99.16
,
24.56
,
49.59
,
199.25
,
99.54
,
149.74
]
elif
wload
==
'wload_4gb'
:
iops
=
[
49.46
,
24.61
,
100.03
,
149.27
,
199.4
,
149.92
]
elif
wload
==
'rreal.4hr'
:
iops
=
[
5.18
,
15.15
,
3.03
,
2.22
,
2.57
,
1.81
,
0.89
,
0.39
]
elif
wload
==
'scr_merg'
:
iops
=
[
3.53
,
0.12
,
51.17
,
55.22
,
362.62
,
0.88
,
212.14
,
11.97
]
elif
wload
==
'real1'
:
iops
=
[
767.01
,
543.30
,
945.15
,
569.38
,
663.33
,
510.78
]
elif
wload
==
'real2'
:
iops
=
[
6.6
,
2.00
,
2.7
,
18.57
,
9.23
]
else
:
iops
=
[
99.16
,
24.56
,
49.59
,
199.25
,
99.54
,
149.74
]
directory
=
basedir_nop
for
i
in
range
(
len
(
prewarmrates
)):
for
j
in
range
(
len
(
prewarmsize
)):
plt
.
clf
()
for
vd
in
range
(
len
(
vdisks
)):
plt
.
scatter
(
iops
[
vd
],
varying_prewarm_rate_per_vdisk
[
vd
][
i
][
j
],
c
=
colors
[
vd
],
label
=
classes
[
vd
],
zorder
=
2
+
vd
)
plt
.
xlabel
(
'IOPS'
)
plt
.
ylabel
(
'5 min average HR'
)
plt
.
legend
(
bbox_to_anchor
=
(
1.01
,
1.0
),
loc
=
'upper left'
,
fancybox
=
True
,
prop
=
{
'size'
:
10
})
plt
.
savefig
(
directory
+
'IopsVariation/'
+
str
(
prewarmsize
[
j
])
+
'
%-
'
+
str
(
prewarmrates
[
i
])
+
'MBps'
,
dpi
=
300
,
bbox_inches
=
'tight'
)
############################################################################################
# SSR = 100K
# SSR = 100K
hatchscale
=
10
hatchscale
=
10
...
@@ -807,7 +1029,8 @@ if __name__ == '__main__':
...
@@ -807,7 +1029,8 @@ if __name__ == '__main__':
# matplotlib.use('agg')
# matplotlib.use('agg')
threads
=
[]
threads
=
[]
wloads
=
[(
'wload_6gb'
,
'6 GiB'
),
(
'wload_4gb'
,
'4 GiB'
),
(
'real1'
,
'6 GiB'
),
(
'scr_merg'
,
'64 MiB'
),
(
'real2'
,
'6 GiB'
)]
wloads
=
[(
'wload_6gb'
,
'6 GiB'
),
(
'wload_4gb'
,
'4 GiB'
),
(
'real1'
,
'6 GiB'
),
(
'scr_merg'
,
'64 MiB'
),
(
'real2'
,
'6 GiB'
)]
wload
=
wloads
[
2
]
# wload = wloads[2]
# wload = wloads[4]
wload
=
(
'wss_iops100'
,
'6 GiB'
)
wload
=
(
'wss_iops100'
,
'6 GiB'
)
# for wload in wloads:
# for wload in wloads:
# x = threading.Thread(target=main, args=(wload[0], wload[1]))
# x = threading.Thread(target=main, args=(wload[0], wload[1]))
...
...
code/run
0 → 100644
View file @
6170e62c
#!/bin/bash
cmake
-H
.
-Bbuild
cmake
--build
build
--
-j3
\ No newline at end of file
code/util.h
View file @
6170e62c
...
@@ -140,7 +140,8 @@ enum PartitionSchemeHM1
...
@@ -140,7 +140,8 @@ enum PartitionSchemeHM1
{
{
HM1_PROP_SHARE
,
HM1_PROP_SHARE
,
HM1_FAIR_SHARE
,
HM1_FAIR_SHARE
,
HM1_NO_PART
HM1_NO_PART
,
HM1_WSS_SHARE
,
};
};
enum
PartitionSchemeHM1HM3
enum
PartitionSchemeHM1HM3
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment