Commit 92761964 authored by Paras Garg's avatar Paras Garg

1. Added JNI

2. Added YSCB basic code
3. Added future for client
4. Added properties in client
parent b9e98c52
......@@ -45,9 +45,10 @@ count:
Jclient: $(OBJS)
$(CXX) -o libHelloImpl.so -shared .build/Hello.o $(LIBS)
$(CXX) -o libhpdosclient.so -L/usr/local/lib -shared $^ $(LIBS)
@echo "jclient "$<" successfully!"
sudo cp libhpdosclient.so /usr/lib
@echo "Copied libhpdosclient.so to /usr/lib"
#jcompile: javac $(JSRCS) -d JBUILD
JniHeader:
......
Steps to build jni client
> make JniHeader <br>
> make JClient <br>
> java -cp jsrc JClient
Running YSCB
> mvn compile <br>
./bin/ycsb load hpdos -P workloads/workloadb -threads 1
mvn -pl site.ycsb:hpdos-binding -am clean package -Dcheckstyle.skip
to do
delete client endpoint on close
threading in client and hashing in client
resolve double delete for get and put
add cache add support for invalidation
interface client API through endpointGroup
Endpointgroup to manage list of servers and caches, Invalidation
./bin/ycsb shell hpdos
./bin/ycsb run hpdos -P workloads/workloada
./bin/ycsb load hpdos -P workloads/workloada
Options:
-P file Specify workload file
-cp path Additional Java classpath entries
-jvm-args args Additional arguments to the JVM
-p key=value Override workload property
-s Print status to stderr
-target n Target ops/sec (default: unthrottled)
-threads n Number of client threads (default: 1)
\ No newline at end of file
# Copyright (c) 2015, 2017 YCSB contributors.
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You
# may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License. See accompanying
# LICENSE file.
# For more info, see: http://EditorConfig.org
root = true
[*.java]
indent_style = space
indent_size = 2
continuation_indent_size = 4
[*.md]
indent_style = space
indent_size = 2
continuation_indent_size = 4
[*.xml]
indent_style = space
indent_size = 2
continuation_indent_size = 4
# ignore compiled byte code
target
# ignore output files from testing
output*
# ignore standard Eclipse files
.project
.classpath
.settings
.checkstyle
# ignore standard IntelliJ files
.idea/
*.iml
*.ipr
*.iws
# ignore standard Vim and Emacs temp files
*.swp
*~
# ignore standard Mac OS X files/dirs
.DS_Store
/differentbin/
# Copyright (c) 2010 Yahoo! Inc., 2012 - 2015 YCSB contributors.
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You
# may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License. See accompanying
# LICENSE file.
# more info here about TravisCI and Java projects
# http://docs.travis-ci.com/user/languages/java/
language: java
jdk:
- openjdk8
- openjdk11
- oraclejdk11
addons:
hosts:
- myshorthost
hostname: myshorthost
postgresql: "9.5"
install:
- mvn -N io.takari:maven:0.7.7:wrapper -Dmaven=3.6.3
- ./mvnw install -q -DskipTests=true
script: ./mvnw test -q
before_script:
- psql -c 'CREATE database test;' -U postgres
- psql -c 'CREATE TABLE usertable (YCSB_KEY VARCHAR(255) PRIMARY KEY not NULL, YCSB_VALUE JSONB not NULL);' -U postgres -d test
- psql -c 'GRANT ALL PRIVILEGES ON DATABASE test to postgres;' -U postgres
# Services to start for tests.
services:
- ignite
- mongodb
- postgresql
# temporarily disable riak. failing, docs offline.
# - riak
# Can't use container based infra because of hosts/hostname
sudo: true
<!--
Copyright (c) 2017 YCSB contributors.
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License. See accompanying
LICENSE file.
-->
## How To Contribute
As more and more databases are created to handle distributed or "cloud" workloads, YCSB needs contributors to write clients to test them. And of course we always need bug fixes, updates for existing databases and new features to keep YCSB going. Here are some guidelines to follow when digging into the code.
## Project Source
YCSB is located in a Git repository hosted on GitHub at [https://github.com/brianfrankcooper/YCSB](https://github.com/brianfrankcooper/YCSB). To modify the code, fork the main repo into your own GitHub account or organization and commit changes there.
YCSB is written in Java (as most of the new cloud data stores at beginning of the project were written in Java) and is laid out as a multi-module Maven project. You should be able to import the project into your favorite IDE or environment easily. For more details about the Maven layout see the [Guide to Working with Multiple Modules](https://maven.apache.org/guides/mini/guide-multiple-modules.html).
## Licensing
YCSB is licensed under the Apache License, Version 2.0 (APL2). Every file included in the project must include the APL header. For example, each Java source file must have a header similar to the following:
```java
/**
* Copyright (c) 2015-2017 YCSB contributors. All rights reserved.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
```
When modifying files that already have a license header, please update the year when you made your edits. E.g. change ``Copyright (c) 2010 Yahoo! Inc., 2012 - 2016 YCSB contributors.`` to ``Copyright (c) 2010 Yahoo! Inc., 2012 - 2017 YCSB contributors.`` If the file only has ``Copyright (c) 2010 Yahoo! Inc.``, append the current year as in ``Copyright (c) 2010 Yahoo! Inc., 2017 YCSB contributors.``.
**WARNING**: It should go without saying, but don't copy and paste code from outside authors or sources. If you are a database author and want to copy some example code, it must be APL2 compatible.
Client bindings to non-APL databases are perfectly acceptable, as data stores are meant to be used from all kinds of projects. Just make sure not to copy any code or commit libraries or binaries into the YCSB code base. Link to them in the Maven pom file.
## Issues and Support
To track bugs, feature requests and releases we use GitHub's integrated [Issues](https://github.com/brianfrankcooper/YCSB/issues). If you find a bug or problem, open an issue with a descriptive title and as many details as you can give us in the body (stack traces, log files, etc). Then if you can create a fix, follow the PR guidelines below.
**Note** Before embarking on a code change or DB, search through the existing issues and pull requests to see if anyone is already working on it. Reach out to them if so.
For general support, please use the mailing list hosted (of course) with Yahoo groups at [http://groups.yahoo.com/group/ycsb-users](http://groups.yahoo.com/group/ycsb-users).
## Code Style
A Java coding style guide is enforced via the Maven CheckStyle plugin. We try not to be too draconian with enforcement but the biggies include:
* Whitespaces instead of tabs.
* Proper Javadocs for methods and classes.
* Camel case member names.
* Upper camel case classes and method names.
* Line length.
CheckStyle will run for pull requests or if you create a package locally so if you just compile and push a commit, you may be surprised when the build fails with a style issue. Just execute ``mvn checkstyle:checkstyle `` before you open a PR and you should avoid any suprises.
## Platforms
Since most data bases aim to support multiple platforms, YCSB aims to run on as many as possible as well. Besides **Linux** and **macOS**, YCSB must compile and run for **Windows**. While not all DBs will run under every platform, the YCSB tool itself must be able to execute on all of these systems and hopefully be able to communicate with remote data stores.
Additionally, YCSB is targeting Java 7 (1.7.0) as its build version as some users are glacially slow moving to Java 8. So please avoid those Lambdas and Streams for now.
## Pull Requests
You've written some amazing code and are excited to share it with the community! It's time to open a PR! Here's what you should do.
* Checkout YCSB's ``master`` branch in your own fork and create a new branch based off of it with a name that is reflective of your work. E.g. ``i123`` for fixing an issue or ``db_xyz`` when working on a binding.
* Add your changes to the branch.
* Commit the code and start the commit message with the component you are working on in square braces. E.g. ``[core] Add another format for exporting histograms.`` or ``[hbase12] Fix interrupted exception bug.``.
* Push to your fork and click the ``Create Pull Request`` button.
* Wait for the build to complete in the CI pipeline. If it fails with a red X, click through the logs for details and fix any issues and commit your changes.
* If you have made changes, please flatten the commits so that the commit logs are nice and clean. Just run a ``git rebase -i <hash before your first commit>``.
After you have opened your PR, a YCSB maintainer will review it and offer constructive feedback via the GitHub review feature. If no one has responded to your PR, please bump the thread by adding comments.
**NOTE**: For maintainers, please get another maintainer to sign off on your changes before merging a PR. And if you're writing code, please do create a PR from your fork, don't just push code directly to the master branch.
## Core, Bindings and Workloads
The main components of the code base include the core library and benchmarking utility, various database client bindings and workload classes and definitions.
### Core
When working on the core classes, keep in mind the following:
* Do not change the core behavior or operation of the main benchmarking classes (Particularly the Client and Workload classes). YCSB is used all over the place because it's a consistent standard that allows different users to compare results with the same workloads. If you find a way to drastically improve throughput, that's great! But please check with the rest of the maintainers to see if we can add the tweaks without invalidating years of benchmarks.
* Do not remove or modify measurements. Users may have tooling to parse the outputs so if you take something out, they'll be a wee bit unhappy. Extending or adding measurements is fine (so if you do have tooling, expect additions.)
* Do not modify existing generators. Again we don't want to invalidate years of benchmarks. Instead, create a new generator or option that can be enabled explicitly (not implicitly!) for users to try out.
* Utility classes and methods are welcome. But if they're only ever used by a specific database binding, co-locate the code with that binding.
* Don't change the DB interface if at all possible. Implementations can squeeze all kinds of workloads through the existing interface and while it may be easy to change the bindings included with the source code, some users may have private clients they can't share with the community.
### Bindings and Clients
When a new database is released a *binding* can be created that implements a client communicating with the given data store that will execute YCSB workloads. Details about writing a DB binding can be found on our [GitHub Wiki page](https://github.com/brianfrankcooper/YCSB/wiki/Adding-a-Database). Some development guidelines to follow include:
* Create a new Maven module for your binding. Follow the existing bindings as examples.
* The module *must* include a README.md file with details such as:
* Database setup with links to documentation so that the YCSB benchmarks will execute properly.
* Example command line executions (workload selection, etc).
* Required and optional properties (e.g. connection strings, behavior settings, etc) along with the default values.
* Versions of the database the binding supports.
* Javadoc the binding and all of the methods. Tell us what it does and how it works.
Because YCSB is a utility to compare multiple data stores, we need each binding to behave similarly by default. That means each data store should enforce the strictest consistency guarantees available and avoid client side buffering or optimizations. This allows users to evaluate different DBs with a common baseline and tough standards.
However you *should* include parameters to tune and improve performance as much as possible to reach those flashy marketing numbers. Just be honest and document what the settings do and what trade-offs are made. (e.g. client side buffering reduces I/O but a crash can lead to data loss).
### Workloads
YCSB began comparing various key/value data stores with simple CRUD operations. However as DBs have become more specialized we've added more workloads for various tasks and would love to have more in the future. Keep the following in mind:
* Make sure more than one publicly available database can handle your workload. It's no fun if only one player is in the game.
* Use the existing DB interface to pass your data around. If you really need another API, discuss with the maintainers to see if there isn't a workaround.
* Provide real-world use cases for the workload, not just theoretical idealizations.
\ No newline at end of file
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
=========================================================================
NOTICE file for use with, and corresponding to Section 4 of,
the Apache License, Version 2.0,
in this case for the YCSB project.
=========================================================================
This product includes software developed by
Yahoo! Inc. (www.yahoo.com)
Copyright (c) 2010 Yahoo! Inc. All rights reserved.
This product includes software developed by
Google Inc. (www.google.com)
Copyright (c) 2015 Google Inc. All rights reserved.
<!--
Copyright (c) 2010 Yahoo! Inc., 2012 - 2016 YCSB contributors.
All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License. See accompanying
LICENSE file.
-->
YCSB
====================================
[![Build Status](https://travis-ci.org/brianfrankcooper/YCSB.png?branch=master)](https://travis-ci.org/brianfrankcooper/YCSB)
Links
-----
* To get here, use https://ycsb.site
* [Our project docs](https://github.com/brianfrankcooper/YCSB/wiki)
* [The original announcement from Yahoo!](https://labs.yahoo.com/news/yahoo-cloud-serving-benchmark/)
Getting Started
---------------
1. Download the [latest release of YCSB](https://github.com/brianfrankcooper/YCSB/releases/latest):
```sh
curl -O --location https://github.com/brianfrankcooper/YCSB/releases/download/0.17.0/ycsb-0.17.0.tar.gz
tar xfvz ycsb-0.17.0.tar.gz
cd ycsb-0.17.0
```
2. Set up a database to benchmark. There is a README file under each binding
directory.
3. Run YCSB command.
On Linux:
```sh
bin/ycsb.sh load basic -P workloads/workloada
bin/ycsb.sh run basic -P workloads/workloada
```
On Windows:
```bat
bin/ycsb.bat load basic -P workloads\workloada
bin/ycsb.bat run basic -P workloads\workloada
```
Running the `ycsb` command without any argument will print the usage.
See https://github.com/brianfrankcooper/YCSB/wiki/Running-a-Workload
for a detailed documentation on how to run a workload.
See https://github.com/brianfrankcooper/YCSB/wiki/Core-Properties for
the list of available workload properties.
Building from source
--------------------
YCSB requires the use of Maven 3; if you use Maven 2, you may see [errors
such as these](https://github.com/brianfrankcooper/YCSB/issues/406).
To build the full distribution, with all database bindings:
mvn clean package
To build a single database binding:
mvn -pl site.ycsb:mongodb-binding -am clean package
<!--
Copyright (c) 2015 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License. See accompanying
LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>site.ycsb</groupId>
<artifactId>root</artifactId>
<version>0.18.0-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>datastore-specific-descriptor</artifactId>
<name>Per Datastore Binding descriptor</name>
<packaging>jar</packaging>
<description>
This module contains the assembly descriptor used by the individual components
to build binding-specific distributions.
</description>
<dependencies>
<dependency>
<groupId>site.ycsb</groupId>
<artifactId>core</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>
<!--
Copyright (c) 2015 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License. See accompanying
LICENSE file.
-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<id>dist</id>
<includeBaseDirectory>true</includeBaseDirectory>
<baseDirectory>ycsb-${artifactId}-${version}</baseDirectory>
<files>
<file>
<source>README.md</source>
<outputDirectory></outputDirectory>
</file>
</files>
<fileSets>
<fileSet>
<directory>..</directory>
<outputDirectory></outputDirectory>
<fileMode>0644</fileMode>
<includes>
<include>LICENSE.txt</include>
<include>NOTICE.txt</include>
</includes>
</fileSet>
<fileSet>
<directory>../bin</directory>
<outputDirectory>bin</outputDirectory>
<fileMode>0755</fileMode>
<includes>
<include>ycsb*</include>
</includes>
</fileSet>
<fileSet>
<directory>../bin</directory>
<outputDirectory>bin</outputDirectory>
<fileMode>0644</fileMode>
<includes>
<include>bindings.properties</include>
</includes>
</fileSet>
<fileSet>
<directory>../workloads</directory>
<outputDirectory>workloads</outputDirectory>
<fileMode>0644</fileMode>
</fileSet>
<fileSet>
<directory>src/main/conf</directory>
<outputDirectory>conf</outputDirectory>
<fileMode>0644</fileMode>
</fileSet>
</fileSets>
<dependencySets>
<dependencySet>
<outputDirectory>lib</outputDirectory>
<includes>
<include>site.ycsb:core</include>
</includes>
<scope>provided</scope>
<useTransitiveFiltering>true</useTransitiveFiltering>
</dependencySet>
<dependencySet>
<outputDirectory>lib</outputDirectory>
<includes>
<include>*:jar:*</include>
</includes>
<excludes>
<exclude>*:sources</exclude>
</excludes>
</dependencySet>
</dependencySets>
</assembly>
<!--
Copyright (c) 2015-2016 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License. See accompanying
LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>site.ycsb</groupId>
<artifactId>root</artifactId>
<version>0.18.0-SNAPSHOT</version>
</parent>
<artifactId>binding-parent</artifactId>
<name>YCSB Datastore Binding Parent</name>
<packaging>pom</packaging>
<description>
This module acts as the parent for new datastore bindings.
It creates a datastore specific binary artifact.
</description>
<modules>
<module>datastore-specific-descriptor</module>
</modules>
<properties>
<!-- See the test-on-jdk9 profile below. Default to 'jdk9 works' -->
<skipJDK9Tests>false</skipJDK9Tests>
<skipJDK10Tests>false</skipJDK10Tests>
<skipJDK11Tests>false</skipJDK11Tests>
</properties>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>${maven.assembly.version}</version>
<dependencies>
<dependency>
<groupId>site.ycsb</groupId>
<artifactId>datastore-specific-descriptor</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<configuration>
<descriptorRefs>
<descriptorRef>datastore-specific-assembly</descriptorRef>
</descriptorRefs>
<finalName>ycsb-${project.artifactId}-${project.version}</finalName>
<formats>
<format>tar.gz</format>
</formats>
<appendAssemblyId>false</appendAssemblyId>
<tarLongFileMode>posix</tarLongFileMode>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<executions>
<execution>
<id>validate</id>
<configuration>
<configLocation>../checkstyle.xml</configLocation>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven.dependency.version}</version>
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>stage-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<!-- If the binding defines a README, presume we should make an assembly. -->
<profile>
<id>datastore-binding</id>
<activation>
<file>
<exists>README.md</exists>
</file>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
</plugin>
</plugins>
</build>
</profile>
<!-- If the binding doesn't work with jdk9, it should redefine the
skipJDK9 property
-->
<profile>
<id>tests-on-jdk9</id>
<activation>
<jdk>9</jdk>
</activation>
<properties>
<skipTests>${skipJDK9Tests}</skipTests>
</properties>
</profile>
<!-- If the binding doesn't work with jdk10, it should redefine the
skipJDK10 property
-->
<profile>
<id>tests-on-jdk10</id>
<activation>
<jdk>10</jdk>
</activation>
<properties>
<skipTests>${skipJDK10Tests}</skipTests>
</properties>
</profile>
<!-- If the binding doesn't work with jdk11, it should redefine the
skipJDK11 property
-->
<profile>
<id>tests-on-jdk11</id>
<activation>
<jdk>11</jdk>
</activation>
<properties>
<skipTests>${skipJDK11Tests}</skipTests>
</properties>
</profile>
<!-- When doing a YCSB release, we want to make sure specific bindings aren't included in the maven repo -->
<profile>
<id>ycsb-release</id>
<properties>
<!-- Set the deploy plugin to skip wherever this property is inherited -->
<maven.deploy.skip>true</maven.deploy.skip>
</properties>
<build>
<plugins>
<!-- We still want to make sure that *this* module gets deployed. -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<executions>
<execution>
<id>but-still-deploy-the-binding-parent</id>
<goals>
<goal>deploy</goal>
</goals>
<phase>deploy</phase>
<inherited>false</inherited>
<configuration>
<skip>false</skip>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
<!--
Copyright (c) 2015 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License. See accompanying
LICENSE file.
-->
# Apache Cassandra 2.x CQL binding
Binding for [Apache Cassandra](http://cassandra.apache.org), using the CQL API
via the [DataStax
driver](http://docs.datastax.com/en/developer/java-driver/2.1/java-driver/whatsNew2.html).
To run against the (deprecated) Cassandra Thrift API, use the `cassandra-10` binding.
## Creating a table for use with YCSB
For keyspace `ycsb`, table `usertable`:
cqlsh> create keyspace ycsb
WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor': 3 };
cqlsh> USE ycsb;
cqlsh> create table usertable (
y_id varchar primary key,
field0 varchar,
field1 varchar,
field2 varchar,
field3 varchar,
field4 varchar,
field5 varchar,
field6 varchar,
field7 varchar,
field8 varchar,
field9 varchar);
**Note that `replication_factor` and consistency levels (below) will affect performance.**
## Cassandra Configuration Parameters
- `hosts` (**required**)
- Cassandra nodes to connect to.
- No default.
* `port`
* CQL port for communicating with Cassandra cluster.
* Default is `9042`.
- `cassandra.keyspace`
Keyspace name - must match the keyspace for the table created (see above).
See http://docs.datastax.com/en/cql/3.1/cql/cql_reference/create_keyspace_r.html for details.
- Default value is `ycsb`
- `cassandra.username`
- `cassandra.password`
- Optional user name and password for authentication. See http://docs.datastax.com/en/cassandra/2.0/cassandra/security/security_config_native_authenticate_t.html for details.
* `cassandra.readconsistencylevel`
* `cassandra.writeconsistencylevel`
* Default value is `QUORUM`
- Consistency level for reads and writes, respectively. See the [DataStax documentation](http://docs.datastax.com/en/cassandra/2.0/cassandra/dml/dml_config_consistency_c.html) for details.
* `cassandra.maxconnections`
* `cassandra.coreconnections`
* Defaults for max and core connections can be found here: https://datastax.github.io/java-driver/2.1.8/features/pooling/#pool-size. Cassandra 2.0.X falls under protocol V2, Cassandra 2.1+ falls under protocol V3.
* `cassandra.connecttimeoutmillis`
* `cassandra.useSSL`
* Default value is false.
- To connect with SSL set this value to true.
* `cassandra.readtimeoutmillis`
* Defaults for connect and read timeouts can be found here: https://docs.datastax.com/en/drivers/java/2.0/com/datastax/driver/core/SocketOptions.html.
* `cassandra.tracing`
* Default is false
* https://docs.datastax.com/en/cql/3.3/cql/cql_reference/tracing_r.html
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (c) 2012-2016 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License. See accompanying
LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>site.ycsb</groupId>
<artifactId>binding-parent</artifactId>
<version>0.18.0-SNAPSHOT</version>
<relativePath>../binding-parent</relativePath>
</parent>
<artifactId>cassandra-binding</artifactId>
<name>Cassandra 2.1+ DB Binding</name>
<packaging>jar</packaging>
<properties>
<!-- Skip tests by default. will be activated by jdk8 profile -->
<skipTests>true</skipTests>
</properties>
<dependencies>
<!-- CQL driver -->
<dependency>
<groupId>com.datastax.cassandra</groupId>
<artifactId>cassandra-driver-core</artifactId>
<version>${cassandra.cql.version}</version>
</dependency>
<dependency>
<groupId>site.ycsb</groupId>
<artifactId>core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.cassandraunit</groupId>
<artifactId>cassandra-unit</artifactId>
<version>3.0.0.1</version>
<classifier>shaded</classifier>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<version>1.7.21</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<!-- only for Cassandra test (Cassandra 2.2+ uses Sigar for collecting system information, and Sigar requires some native lib files) -->
<dependency>
<groupId>org.hyperic</groupId>
<artifactId>sigar-dist</artifactId>
<version>1.6.4.129</version>
<type>zip</type>
<scope>test</scope>
</dependency>
</dependencies>
<profiles>
<!-- Cassandra 2.2+ requires JDK8 to run, so none of our tests
will work unless we're using jdk8.
-->
<profile>
<id>jdk8-tests</id>
<activation>
<jdk>1.8</jdk>
</activation>
<properties>
<skipTests>false</skipTests>
</properties>
</profile>
</profiles>
<!-- sigar-dist can be downloaded from jboss repository -->
<repositories>
<repository>
<id>central2</id>
<name>sigar Repository</name>
<url>https://repository.jboss.org/nexus/content/groups/public-jboss/</url>
<layout>default</layout>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
<!-- unzip sigar-dist/lib files.
References:
http://stackoverflow.com/questions/5388661/unzip-dependency-in-maven
https://arviarya.wordpress.com/2013/09/22/sigar-access-operating-system-and-hardware-level-information/
-->
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>unpack-sigar</id>
<phase>process-test-resources<!-- or any other valid maven phase --></phase>
<goals>
<goal>unpack-dependencies</goal>
</goals>
<configuration>
<includeGroupIds>org.hyperic</includeGroupIds>
<includeArtifactIds>sigar-dist</includeArtifactIds>
<includes>**/sigar-bin/lib/*</includes>
<excludes>**/sigar-bin/lib/*jar</excludes>
<outputDirectory>
${project.build.directory}/cassandra-dependency
</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.8</version>
<configuration>
<argLine>-Djava.library.path=${project.build.directory}/cassandra-dependency/hyperic-sigar-1.6.4/sigar-bin/lib</argLine>
</configuration>
</plugin>
</plugins>
</build>
</project>
/**
* Copyright (c) 2013-2015 YCSB contributors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License. See accompanying LICENSE file.
*
* Submitted by Chrisjan Matser on 10/11/2010.
*/
package site.ycsb.db;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.ColumnDefinitions;
import com.datastax.driver.core.ConsistencyLevel;
import com.datastax.driver.core.Host;
import com.datastax.driver.core.HostDistance;
import com.datastax.driver.core.Metadata;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.querybuilder.Insert;
import com.datastax.driver.core.querybuilder.QueryBuilder;
import com.datastax.driver.core.querybuilder.Select;
import com.datastax.driver.core.querybuilder.Update;
import site.ycsb.ByteArrayByteIterator;
import site.ycsb.ByteIterator;
import site.ycsb.DB;
import site.ycsb.DBException;
import site.ycsb.Status;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.Vector;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.helpers.MessageFormatter;
/**
* Cassandra 2.x CQL client.
*
* See {@code cassandra2/README.md} for details.
*
* @author cmatser
*/
public class CassandraCQLClient extends DB {
private static Logger logger = LoggerFactory.getLogger(CassandraCQLClient.class);
private static Cluster cluster = null;
private static Session session = null;
private static ConcurrentMap<Set<String>, PreparedStatement> readStmts =
new ConcurrentHashMap<Set<String>, PreparedStatement>();
private static ConcurrentMap<Set<String>, PreparedStatement> scanStmts =
new ConcurrentHashMap<Set<String>, PreparedStatement>();
private static ConcurrentMap<Set<String>, PreparedStatement> insertStmts =
new ConcurrentHashMap<Set<String>, PreparedStatement>();
private static ConcurrentMap<Set<String>, PreparedStatement> updateStmts =
new ConcurrentHashMap<Set<String>, PreparedStatement>();
private static AtomicReference<PreparedStatement> readAllStmt =
new AtomicReference<PreparedStatement>();
private static AtomicReference<PreparedStatement> scanAllStmt =
new AtomicReference<PreparedStatement>();
private static AtomicReference<PreparedStatement> deleteStmt =
new AtomicReference<PreparedStatement>();
private static ConsistencyLevel readConsistencyLevel = ConsistencyLevel.QUORUM;
private static ConsistencyLevel writeConsistencyLevel = ConsistencyLevel.QUORUM;
public static final String YCSB_KEY = "y_id";
public static final String KEYSPACE_PROPERTY = "cassandra.keyspace";
public static final String KEYSPACE_PROPERTY_DEFAULT = "ycsb";
public static final String USERNAME_PROPERTY = "cassandra.username";
public static final String PASSWORD_PROPERTY = "cassandra.password";
public static final String HOSTS_PROPERTY = "hosts";
public static final String PORT_PROPERTY = "port";
public static final String PORT_PROPERTY_DEFAULT = "9042";
public static final String READ_CONSISTENCY_LEVEL_PROPERTY =
"cassandra.readconsistencylevel";
public static final String READ_CONSISTENCY_LEVEL_PROPERTY_DEFAULT = readConsistencyLevel.name();
public static final String WRITE_CONSISTENCY_LEVEL_PROPERTY =
"cassandra.writeconsistencylevel";
public static final String WRITE_CONSISTENCY_LEVEL_PROPERTY_DEFAULT = writeConsistencyLevel.name();
public static final String MAX_CONNECTIONS_PROPERTY =
"cassandra.maxconnections";
public static final String CORE_CONNECTIONS_PROPERTY =
"cassandra.coreconnections";
public static final String CONNECT_TIMEOUT_MILLIS_PROPERTY =
"cassandra.connecttimeoutmillis";
public static final String READ_TIMEOUT_MILLIS_PROPERTY =
"cassandra.readtimeoutmillis";
public static final String TRACING_PROPERTY = "cassandra.tracing";
public static final String TRACING_PROPERTY_DEFAULT = "false";
public static final String USE_SSL_CONNECTION = "cassandra.useSSL";
private static final String DEFAULT_USE_SSL_CONNECTION = "false";
/**
* Count the number of times initialized to teardown on the last
* {@link #cleanup()}.
*/
private static final AtomicInteger INIT_COUNT = new AtomicInteger(0);
private static boolean debug = false;
private static boolean trace = false;
/**
* Initialize any state for this DB. Called once per DB instance; there is one
* DB instance per client thread.
*/
@Override
public void init() throws DBException {
// Keep track of number of calls to init (for later cleanup)
INIT_COUNT.incrementAndGet();
// Synchronized so that we only have a single
// cluster/session instance for all the threads.
synchronized (INIT_COUNT) {
// Check if the cluster has already been initialized
if (cluster != null) {
return;
}
try {
debug =
Boolean.parseBoolean(getProperties().getProperty("debug", "false"));
trace = Boolean.valueOf(getProperties().getProperty(TRACING_PROPERTY, TRACING_PROPERTY_DEFAULT));
String host = getProperties().getProperty(HOSTS_PROPERTY);
if (host == null) {
throw new DBException(String.format(
"Required property \"%s\" missing for CassandraCQLClient",
HOSTS_PROPERTY));
}
String[] hosts = host.split(",");
String port = getProperties().getProperty(PORT_PROPERTY, PORT_PROPERTY_DEFAULT);
String username = getProperties().getProperty(USERNAME_PROPERTY);
String password = getProperties().getProperty(PASSWORD_PROPERTY);
String keyspace = getProperties().getProperty(KEYSPACE_PROPERTY,
KEYSPACE_PROPERTY_DEFAULT);
readConsistencyLevel = ConsistencyLevel.valueOf(
getProperties().getProperty(READ_CONSISTENCY_LEVEL_PROPERTY,
READ_CONSISTENCY_LEVEL_PROPERTY_DEFAULT));
writeConsistencyLevel = ConsistencyLevel.valueOf(
getProperties().getProperty(WRITE_CONSISTENCY_LEVEL_PROPERTY,
WRITE_CONSISTENCY_LEVEL_PROPERTY_DEFAULT));
Boolean useSSL = Boolean.parseBoolean(getProperties().getProperty(USE_SSL_CONNECTION,
DEFAULT_USE_SSL_CONNECTION));
if ((username != null) && !username.isEmpty()) {
Cluster.Builder clusterBuilder = Cluster.builder().withCredentials(username, password)
.withPort(Integer.valueOf(port)).addContactPoints(hosts);
if (useSSL) {
clusterBuilder = clusterBuilder.withSSL();
}
cluster = clusterBuilder.build();
} else {
cluster = Cluster.builder().withPort(Integer.valueOf(port))
.addContactPoints(hosts).build();
}
String maxConnections = getProperties().getProperty(
MAX_CONNECTIONS_PROPERTY);
if (maxConnections != null) {
cluster.getConfiguration().getPoolingOptions()
.setMaxConnectionsPerHost(HostDistance.LOCAL,
Integer.valueOf(maxConnections));
}
String coreConnections = getProperties().getProperty(
CORE_CONNECTIONS_PROPERTY);
if (coreConnections != null) {
cluster.getConfiguration().getPoolingOptions()
.setCoreConnectionsPerHost(HostDistance.LOCAL,
Integer.valueOf(coreConnections));
}
String connectTimoutMillis = getProperties().getProperty(
CONNECT_TIMEOUT_MILLIS_PROPERTY);
if (connectTimoutMillis != null) {
cluster.getConfiguration().getSocketOptions()
.setConnectTimeoutMillis(Integer.valueOf(connectTimoutMillis));
}
String readTimoutMillis = getProperties().getProperty(
READ_TIMEOUT_MILLIS_PROPERTY);
if (readTimoutMillis != null) {
cluster.getConfiguration().getSocketOptions()
.setReadTimeoutMillis(Integer.valueOf(readTimoutMillis));
}
Metadata metadata = cluster.getMetadata();
logger.info("Connected to cluster: {}\n",
metadata.getClusterName());
for (Host discoveredHost : metadata.getAllHosts()) {
logger.info("Datacenter: {}; Host: {}; Rack: {}\n",
discoveredHost.getDatacenter(), discoveredHost.getAddress(),
discoveredHost.getRack());
}
session = cluster.connect(keyspace);
} catch (Exception e) {
throw new DBException(e);
}
} // synchronized
}
/**
* Cleanup any state for this DB. Called once per DB instance; there is one DB
* instance per client thread.
*/
@Override
public void cleanup() throws DBException {
synchronized (INIT_COUNT) {
final int curInitCount = INIT_COUNT.decrementAndGet();
if (curInitCount <= 0) {
readStmts.clear();
scanStmts.clear();
insertStmts.clear();
updateStmts.clear();
readAllStmt.set(null);
scanAllStmt.set(null);
deleteStmt.set(null);
session.close();
cluster.close();
cluster = null;
session = null;
}
if (curInitCount < 0) {
// This should never happen.
throw new DBException(
String.format("initCount is negative: %d", curInitCount));
}
}
}
/**
* Read a record from the database. Each field/value pair from the result will
* be stored in a HashMap.
*
* @param table
* The name of the table
* @param key
* The record key of the record to read.
* @param fields
* The list of fields to read, or null for all of them
* @param result
* A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error
*/
@Override
public Status read(String table, String key, Set<String> fields,
Map<String, ByteIterator> result) {
try {
PreparedStatement stmt = (fields == null) ? readAllStmt.get() : readStmts.get(fields);
// Prepare statement on demand
if (stmt == null) {
Select.Builder selectBuilder;
if (fields == null) {
selectBuilder = QueryBuilder.select().all();
} else {
selectBuilder = QueryBuilder.select();
for (String col : fields) {
((Select.Selection) selectBuilder).column(col);
}
}
stmt = session.prepare(selectBuilder.from(table)
.where(QueryBuilder.eq(YCSB_KEY, QueryBuilder.bindMarker()))
.limit(1));
stmt.setConsistencyLevel(readConsistencyLevel);
if (trace) {
stmt.enableTracing();
}
PreparedStatement prevStmt = (fields == null) ?
readAllStmt.getAndSet(stmt) :
readStmts.putIfAbsent(new HashSet(fields), stmt);
if (prevStmt != null) {
stmt = prevStmt;
}
}
logger.debug(stmt.getQueryString());
logger.debug("key = {}", key);
ResultSet rs = session.execute(stmt.bind(key));
if (rs.isExhausted()) {
return Status.NOT_FOUND;
}
// Should be only 1 row
Row row = rs.one();
ColumnDefinitions cd = row.getColumnDefinitions();
for (ColumnDefinitions.Definition def : cd) {
ByteBuffer val = row.getBytesUnsafe(def.getName());
if (val != null) {
result.put(def.getName(), new ByteArrayByteIterator(val.array()));
} else {
result.put(def.getName(), null);
}
}
return Status.OK;
} catch (Exception e) {
logger.error(MessageFormatter.format("Error reading key: {}", key).getMessage(), e);
return Status.ERROR;
}
}
/**
* Perform a range scan for a set of records in the database. Each field/value
* pair from the result will be stored in a HashMap.
*
* Cassandra CQL uses "token" method for range scan which doesn't always yield
* intuitive results.
*
* @param table
* The name of the table
* @param startkey
* The record key of the first record to read.
* @param recordcount
* The number of records to read
* @param fields
* The list of fields to read, or null for all of them
* @param result
* A Vector of HashMaps, where each HashMap is a set field/value
* pairs for one record
* @return Zero on success, a non-zero error code on error
*/
@Override
public Status scan(String table, String startkey, int recordcount,
Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
try {
PreparedStatement stmt = (fields == null) ? scanAllStmt.get() : scanStmts.get(fields);
// Prepare statement on demand
if (stmt == null) {
Select.Builder selectBuilder;
if (fields == null) {
selectBuilder = QueryBuilder.select().all();
} else {
selectBuilder = QueryBuilder.select();
for (String col : fields) {
((Select.Selection) selectBuilder).column(col);
}
}
Select selectStmt = selectBuilder.from(table);
// The statement builder is not setup right for tokens.
// So, we need to build it manually.
String initialStmt = selectStmt.toString();
StringBuilder scanStmt = new StringBuilder();
scanStmt.append(initialStmt.substring(0, initialStmt.length() - 1));
scanStmt.append(" WHERE ");
scanStmt.append(QueryBuilder.token(YCSB_KEY));
scanStmt.append(" >= ");
scanStmt.append("token(");
scanStmt.append(QueryBuilder.bindMarker());
scanStmt.append(")");
scanStmt.append(" LIMIT ");
scanStmt.append(QueryBuilder.bindMarker());
stmt = session.prepare(scanStmt.toString());
stmt.setConsistencyLevel(readConsistencyLevel);
if (trace) {
stmt.enableTracing();
}
PreparedStatement prevStmt = (fields == null) ?
scanAllStmt.getAndSet(stmt) :
scanStmts.putIfAbsent(new HashSet(fields), stmt);
if (prevStmt != null) {
stmt = prevStmt;
}
}
logger.debug(stmt.getQueryString());
logger.debug("startKey = {}, recordcount = {}", startkey, recordcount);
ResultSet rs = session.execute(stmt.bind(startkey, Integer.valueOf(recordcount)));
HashMap<String, ByteIterator> tuple;
while (!rs.isExhausted()) {
Row row = rs.one();
tuple = new HashMap<String, ByteIterator>();
ColumnDefinitions cd = row.getColumnDefinitions();
for (ColumnDefinitions.Definition def : cd) {
ByteBuffer val = row.getBytesUnsafe(def.getName());
if (val != null) {
tuple.put(def.getName(), new ByteArrayByteIterator(val.array()));
} else {
tuple.put(def.getName(), null);
}
}
result.add(tuple);
}
return Status.OK;
} catch (Exception e) {
logger.error(
MessageFormatter.format("Error scanning with startkey: {}", startkey).getMessage(), e);
return Status.ERROR;
}
}
/**
* Update a record in the database. Any field/value pairs in the specified
* values HashMap will be written into the record with the specified record
* key, overwriting any existing values with the same field name.
*
* @param table
* The name of the table
* @param key
* The record key of the record to write.
* @param values
* A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error
*/
@Override
public Status update(String table, String key, Map<String, ByteIterator> values) {
try {
Set<String> fields = values.keySet();
PreparedStatement stmt = updateStmts.get(fields);
// Prepare statement on demand
if (stmt == null) {
Update updateStmt = QueryBuilder.update(table);
// Add fields
for (String field : fields) {
updateStmt.with(QueryBuilder.set(field, QueryBuilder.bindMarker()));
}
// Add key
updateStmt.where(QueryBuilder.eq(YCSB_KEY, QueryBuilder.bindMarker()));
stmt = session.prepare(updateStmt);
stmt.setConsistencyLevel(writeConsistencyLevel);
if (trace) {
stmt.enableTracing();
}
PreparedStatement prevStmt = updateStmts.putIfAbsent(new HashSet(fields), stmt);
if (prevStmt != null) {
stmt = prevStmt;
}
}
if (logger.isDebugEnabled()) {
logger.debug(stmt.getQueryString());
logger.debug("key = {}", key);
for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
logger.debug("{} = {}", entry.getKey(), entry.getValue());
}
}
// Add fields
ColumnDefinitions vars = stmt.getVariables();
BoundStatement boundStmt = stmt.bind();
for (int i = 0; i < vars.size() - 1; i++) {
boundStmt.setString(i, values.get(vars.getName(i)).toString());
}
// Add key
boundStmt.setString(vars.size() - 1, key);
session.execute(boundStmt);
return Status.OK;
} catch (Exception e) {
logger.error(MessageFormatter.format("Error updating key: {}", key).getMessage(), e);
}
return Status.ERROR;
}
/**
* Insert a record in the database. Any field/value pairs in the specified
* values HashMap will be written into the record with the specified record
* key.
*
* @param table
* The name of the table
* @param key
* The record key of the record to insert.
* @param values
* A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error
*/
@Override
public Status insert(String table, String key, Map<String, ByteIterator> values) {
try {
Set<String> fields = values.keySet();
PreparedStatement stmt = insertStmts.get(fields);
// Prepare statement on demand
if (stmt == null) {
Insert insertStmt = QueryBuilder.insertInto(table);
// Add key
insertStmt.value(YCSB_KEY, QueryBuilder.bindMarker());
// Add fields
for (String field : fields) {
insertStmt.value(field, QueryBuilder.bindMarker());
}
stmt = session.prepare(insertStmt);
stmt.setConsistencyLevel(writeConsistencyLevel);
if (trace) {
stmt.enableTracing();
}
PreparedStatement prevStmt = insertStmts.putIfAbsent(new HashSet(fields), stmt);
if (prevStmt != null) {
stmt = prevStmt;
}
}
if (logger.isDebugEnabled()) {
logger.debug(stmt.getQueryString());
logger.debug("key = {}", key);
for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
logger.debug("{} = {}", entry.getKey(), entry.getValue());
}
}
// Add key
BoundStatement boundStmt = stmt.bind().setString(0, key);
// Add fields
ColumnDefinitions vars = stmt.getVariables();
for (int i = 1; i < vars.size(); i++) {
boundStmt.setString(i, values.get(vars.getName(i)).toString());
}
session.execute(boundStmt);
return Status.OK;
} catch (Exception e) {
logger.error(MessageFormatter.format("Error inserting key: {}", key).getMessage(), e);
}
return Status.ERROR;
}
/**
* Delete a record from the database.
*
* @param table
* The name of the table
* @param key
* The record key of the record to delete.
* @return Zero on success, a non-zero error code on error
*/
@Override
public Status delete(String table, String key) {
try {
PreparedStatement stmt = deleteStmt.get();
// Prepare statement on demand
if (stmt == null) {
stmt = session.prepare(QueryBuilder.delete().from(table)
.where(QueryBuilder.eq(YCSB_KEY, QueryBuilder.bindMarker())));
stmt.setConsistencyLevel(writeConsistencyLevel);
if (trace) {
stmt.enableTracing();
}
PreparedStatement prevStmt = deleteStmt.getAndSet(stmt);
if (prevStmt != null) {
stmt = prevStmt;
}
}
logger.debug(stmt.getQueryString());
logger.debug("key = {}", key);
session.execute(stmt.bind(key));
return Status.OK;
} catch (Exception e) {
logger.error(MessageFormatter.format("Error deleting key: {}", key).getMessage(), e);
}
return Status.ERROR;
}
}
/*
* Copyright (c) 2014, Yahoo!, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
/**
* The YCSB binding for <a href="http://cassandra.apache.org/">Cassandra</a>
* 2.1+ via CQL.
*/
package site.ycsb.db;
/**
* Copyright (c) 2015 YCSB contributors All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package site.ycsb.db;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.hasEntry;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import com.google.common.collect.Sets;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.Statement;
import com.datastax.driver.core.querybuilder.Insert;
import com.datastax.driver.core.querybuilder.QueryBuilder;
import com.datastax.driver.core.querybuilder.Select;
import site.ycsb.ByteIterator;
import site.ycsb.Status;
import site.ycsb.StringByteIterator;
import site.ycsb.measurements.Measurements;
import site.ycsb.workloads.CoreWorkload;
import org.cassandraunit.CassandraCQLUnit;
import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
/**
* Integration tests for the Cassandra client
*/
public class CassandraCQLClientTest {
// Change the default Cassandra timeout from 10s to 120s for slow CI machines
private final static long timeout = 120000L;
private final static String TABLE = "usertable";
private final static String HOST = "localhost";
private final static int PORT = 9142;
private final static String DEFAULT_ROW_KEY = "user1";
private CassandraCQLClient client;
private Session session;
@ClassRule
public static CassandraCQLUnit cassandraUnit = new CassandraCQLUnit(
new ClassPathCQLDataSet("ycsb.cql", "ycsb"), null, timeout);
@Before
public void setUp() throws Exception {
session = cassandraUnit.getSession();
Properties p = new Properties();
p.setProperty("hosts", HOST);
p.setProperty("port", Integer.toString(PORT));
p.setProperty("table", TABLE);
Measurements.setProperties(p);
final CoreWorkload workload = new CoreWorkload();
workload.init(p);
client = new CassandraCQLClient();
client.setProperties(p);
client.init();
}
@After
public void tearDownClient() throws Exception {
if (client != null) {
client.cleanup();
}
client = null;
}
@After
public void clearTable() throws Exception {
// Clear the table so that each test starts fresh.
final Statement truncate = QueryBuilder.truncate(TABLE);
if (cassandraUnit != null) {
cassandraUnit.getSession().execute(truncate);
}
}
@Test
public void testReadMissingRow() throws Exception {
final HashMap<String, ByteIterator> result = new HashMap<String, ByteIterator>();
final Status status = client.read(TABLE, "Missing row", null, result);
assertThat(result.size(), is(0));
assertThat(status, is(Status.NOT_FOUND));
}
private void insertRow() {
final String rowKey = DEFAULT_ROW_KEY;
Insert insertStmt = QueryBuilder.insertInto(TABLE);
insertStmt.value(CassandraCQLClient.YCSB_KEY, rowKey);
insertStmt.value("field0", "value1");
insertStmt.value("field1", "value2");
session.execute(insertStmt);
}
@Test
public void testRead() throws Exception {
insertRow();
final HashMap<String, ByteIterator> result = new HashMap<String, ByteIterator>();
final Status status = client.read(TABLE, DEFAULT_ROW_KEY, null, result);
assertThat(status, is(Status.OK));
assertThat(result.entrySet(), hasSize(11));
assertThat(result, hasEntry("field2", null));
final HashMap<String, String> strResult = new HashMap<String, String>();
for (final Map.Entry<String, ByteIterator> e : result.entrySet()) {
if (e.getValue() != null) {
strResult.put(e.getKey(), e.getValue().toString());
}
}
assertThat(strResult, hasEntry(CassandraCQLClient.YCSB_KEY, DEFAULT_ROW_KEY));
assertThat(strResult, hasEntry("field0", "value1"));
assertThat(strResult, hasEntry("field1", "value2"));
}
@Test
public void testReadSingleColumn() throws Exception {
insertRow();
final HashMap<String, ByteIterator> result = new HashMap<String, ByteIterator>();
final Set<String> fields = Sets.newHashSet("field1");
final Status status = client.read(TABLE, DEFAULT_ROW_KEY, fields, result);
assertThat(status, is(Status.OK));
assertThat(result.entrySet(), hasSize(1));
final Map<String, String> strResult = StringByteIterator.getStringMap(result);
assertThat(strResult, hasEntry("field1", "value2"));
}
@Test
public void testInsert() throws Exception {
final String key = "key";
final Map<String, String> input = new HashMap<String, String>();
input.put("field0", "value1");
input.put("field1", "value2");
final Status status = client.insert(TABLE, key, StringByteIterator.getByteIteratorMap(input));
assertThat(status, is(Status.OK));
// Verify result
final Select selectStmt =
QueryBuilder.select("field0", "field1")
.from(TABLE)
.where(QueryBuilder.eq(CassandraCQLClient.YCSB_KEY, key))
.limit(1);
final ResultSet rs = session.execute(selectStmt);
final Row row = rs.one();
assertThat(row, notNullValue());
assertThat(rs.isExhausted(), is(true));
assertThat(row.getString("field0"), is("value1"));
assertThat(row.getString("field1"), is("value2"));
}
@Test
public void testUpdate() throws Exception {
insertRow();
final Map<String, String> input = new HashMap<String, String>();
input.put("field0", "new-value1");
input.put("field1", "new-value2");
final Status status = client.update(TABLE,
DEFAULT_ROW_KEY,
StringByteIterator.getByteIteratorMap(input));
assertThat(status, is(Status.OK));
// Verify result
final Select selectStmt =
QueryBuilder.select("field0", "field1")
.from(TABLE)
.where(QueryBuilder.eq(CassandraCQLClient.YCSB_KEY, DEFAULT_ROW_KEY))
.limit(1);
final ResultSet rs = session.execute(selectStmt);
final Row row = rs.one();
assertThat(row, notNullValue());
assertThat(rs.isExhausted(), is(true));
assertThat(row.getString("field0"), is("new-value1"));
assertThat(row.getString("field1"), is("new-value2"));
}
@Test
public void testDelete() throws Exception {
insertRow();
final Status status = client.delete(TABLE, DEFAULT_ROW_KEY);
assertThat(status, is(Status.OK));
// Verify result
final Select selectStmt =
QueryBuilder.select("field0", "field1")
.from(TABLE)
.where(QueryBuilder.eq(CassandraCQLClient.YCSB_KEY, DEFAULT_ROW_KEY))
.limit(1);
final ResultSet rs = session.execute(selectStmt);
final Row row = rs.one();
assertThat(row, nullValue());
}
@Test
public void testPreparedStatements() throws Exception {
final int LOOP_COUNT = 3;
for (int i = 0; i < LOOP_COUNT; i++) {
testInsert();
testUpdate();
testRead();
testReadSingleColumn();
testReadMissingRow();
testDelete();
}
}
}
/**
* Copyright (c) 2015 YCSB Contributors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
CREATE TABLE usertable (
y_id varchar primary key,
field0 varchar,
field1 varchar,
field2 varchar,
field3 varchar,
field4 varchar,
field5 varchar,
field6 varchar,
field7 varchar,
field8 varchar,
field9 varchar);
<?xml version="1.0"?>
<!--
Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License. See accompanying
LICENSE file.
-->
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.2//EN"
"http://www.puppycrawl.com/dtds/configuration_1_2.dtd">
<!--
Checkstyle configuration for Hadoop that is based on the sun_checks.xml file
that is bundled with Checkstyle and includes checks for:
- the Java Language Specification at
http://java.sun.com/docs/books/jls/second_edition/html/index.html
- the Sun Code Conventions at http://java.sun.com/docs/codeconv/
- the Javadoc guidelines at
http://java.sun.com/j2se/javadoc/writingdoccomments/index.html
- the JDK Api documentation http://java.sun.com/j2se/docs/api/index.html
- some best practices
Checkstyle is very configurable. Be sure to read the documentation at
http://checkstyle.sf.net (or in your downloaded distribution).
Most Checks are configurable, be sure to consult the documentation.
To completely disable a check, just comment it out or delete it from the file.
Finally, it is worth reading the documentation.
-->
<module name="Checker">
<!-- Checks that a package.html file exists for each package. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html#PackageHtml -->
<module name="JavadocPackage"/>
<!-- Checks whether files end with a new line. -->
<!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile -->
<!-- module name="NewlineAtEndOfFile"/-->
<!-- Checks that property files contain the same keys. -->
<!-- See http://checkstyle.sf.net/config_misc.html#Translation -->
<module name="Translation"/>
<module name="FileLength"/>
<module name="FileTabCharacter"/>
<module name="TreeWalker">
<!-- Checks for Javadoc comments. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html -->
<module name="JavadocType">
<property name="scope" value="public"/>
<property name="allowMissingParamTags" value="true"/>
<!-- unfortunately we cannot add implNote, implSpec, apiNote and apiSpec to checkstyle -->
<property name="allowUnknownTags" value="true"/>
</module>
<module name="JavadocStyle"/>
<!-- Checks for Naming Conventions. -->
<!-- See http://checkstyle.sf.net/config_naming.html -->
<module name="ConstantName"/>
<module name="LocalFinalVariableName"/>
<module name="LocalVariableName"/>
<module name="MemberName"/>
<module name="MethodName"/>
<module name="PackageName"/>
<module name="ParameterName"/>
<module name="StaticVariableName"/>
<module name="TypeName"/>
<!-- Checks for Headers -->
<!-- See http://checkstyle.sf.net/config_header.html -->
<!-- <module name="Header"> -->
<!-- The follow property value demonstrates the ability -->
<!-- to have access to ANT properties. In this case it uses -->
<!-- the ${basedir} property to allow Checkstyle to be run -->
<!-- from any directory within a project. See property -->
<!-- expansion, -->
<!-- http://checkstyle.sf.net/config.html#properties -->
<!-- <property -->
<!-- name="headerFile" -->
<!-- value="${basedir}/java.header"/> -->
<!-- </module> -->
<!-- Following interprets the header file as regular expressions. -->
<!-- <module name="RegexpHeader"/> -->
<!-- Checks for imports -->
<!-- See http://checkstyle.sf.net/config_import.html -->
<module name="IllegalImport"/> <!-- defaults to sun.* packages -->
<module name="RedundantImport"/>
<module name="UnusedImports"/>
<!-- Checks for Size Violations. -->
<!-- See http://checkstyle.sf.net/config_sizes.html -->
<module name="LineLength">
<property name="max" value="120"/>
</module>
<module name="MethodLength"/>
<module name="ParameterNumber"/>
<!-- Checks for whitespace -->
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
<module name="EmptyForIteratorPad"/>
<module name="MethodParamPad"/>
<module name="NoWhitespaceAfter"/>
<module name="NoWhitespaceBefore"/>
<module name="ParenPad"/>
<module name="TypecastParenPad"/>
<module name="WhitespaceAfter">
<property name="tokens" value="COMMA, SEMI"/>
</module>
<!-- Modifier Checks -->
<!-- See http://checkstyle.sf.net/config_modifiers.html -->
<module name="ModifierOrder"/>
<module name="RedundantModifier"/>
<!-- Checks for blocks. You know, those {}'s -->
<!-- See http://checkstyle.sf.net/config_blocks.html -->
<module name="AvoidNestedBlocks"/>
<module name="EmptyBlock">
<property name="option" value="text"/>
</module>
<module name="LeftCurly"/>
<module name="NeedBraces"/>
<module name="RightCurly"/>
<!-- Checks for common coding problems -->
<!-- See http://checkstyle.sf.net/config_coding.html -->
<!-- module name="AvoidInlineConditionals"/-->
<module name="EmptyStatement"/>
<module name="EqualsHashCode"/>
<module name="HiddenField">
<property name="ignoreConstructorParameter" value="true"/>
</module>
<module name="IllegalInstantiation"/>
<module name="InnerAssignment"/>
<module name="MissingSwitchDefault"/>
<module name="SimplifyBooleanExpression"/>
<module name="SimplifyBooleanReturn"/>
<!-- Checks for class design -->
<!-- See http://checkstyle.sf.net/config_design.html -->
<module name="FinalClass"/>
<module name="HideUtilityClassConstructor"/>
<module name="InterfaceIsType"/>
<module name="VisibilityModifier">
<property name="protectedAllowed" value="true"/>
</module>
<!-- Miscellaneous other checks. -->
<!-- See http://checkstyle.sf.net/config_misc.html -->
<module name="ArrayTypeStyle"/>
<module name="Indentation">
<property name="basicOffset" value="2" />
<property name="caseIndent" value="0" />
</module>
<!-- <module name="TodoComment"/> -->
<module name="UpperEll"/>
</module>
</module>
<!--
Copyright (c) 2015 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License. See accompanying
LICENSE file.
-->
When used as a latency under load benchmark YCSB in it's original form suffers from
Coordinated Omission[1] and related measurement issue:
* Load is controlled by response time
* Measurement does not account for missing time
* Measurement starts at beginning of request rather than at intended beginning
* Measurement is limited in scope as the histogram does not provide data on overflow values
To provide a minimal correction patch the following were implemented:
1. Replace internal histogram implementation with HdrHistogram[2]:
HdrHistogram offers a dynamic range of measurement at a given precision and will
improve the fidelity of reporting. It allows capturing a much wider range of latencies.
HdrHistogram also supports compressed loss-less serialization which enable capturing
snapshot histograms from which lower resolution histograms can be constructed for plotting
latency over time. Snapshot interval histograms are serialized on status reporting which
must be enabled using the '-s' option.
2. Track intended operation start and report latencies from that point in time:
Assuming the benchmark sets a target schedule of execution in which every operation
is supposed to happen at a given time the benchmark should measure the latency between
intended start time and operation completion.
This required the introduction of a new measurement point and inevitably
includes measuring some of the internal preparation steps of the load generator.
These overhead should be negligible in the context of a network hop, but could
be corrected for by estimating the load-generator overheads (e.g. by measuring a
no-op DB or by measuring the setup time for an operation and deducting that from total).
This intended measurement point is only used when there is a target load (specified by
the -target paramaeter)
This branch supports the following new options:
* -p measurementtype=[histogram|hdrhistogram|hdrhistogram+histogram|timeseries] (default=histogram)
The new measurement types are hdrhistogram and hdrhistogram+histogram. Default is still
histogram, which is the old histogram. Ultimately we would remove the old measurement types
and use only HdrHistogram but the old measurement is left in there for comparison sake.
* -p measurement.interval=[op|intended|both] (default=op)
This new option deferentiates between measured intervals and adds the intended interval(as described)
above, and the option to record both the op and intended for comparison.
* -p hdrhistogram.fileoutput=[true|false] (default=false)
This new option will enable periodical writes of the interval histogram into an output file. The path can be set using '-p hdrhistogram.output.path=<PATH>'.
Example parameters:
-target 1000 -s -p workload=site.ycsb.workloads.CoreWorkload -p basicdb.verbose=false -p basicdb.simulatedelay=4 -p measurement.interval=both -p measurementtype=hdrhistogram -p hdrhistogram.fileoutput=true -p maxexecutiontime=60
Further changes made:
* -p status.interval=<number of seconds> (default=10)
Controls the number of seconds between status reports and therefore between HdrHistogram snapshots reported.
* -p basicdb.randomizedelay=[true|false] (default=true)
Controls weather the delay simulated by the mock DB is uniformly random or not.
Further suggestions:
1. Correction load control: currently after a pause the load generator will do
operations back to back to catchup, this leads to a flat out throughput mode
of testing as opposed to controlled load.
2. Move to async model: Scenarios where Ops have no dependency could delegate the
Op execution to a threadpool and thus separate the request rate control from the
synchronous execution of Ops. Measurement would start on queuing for execution.
1. https://groups.google.com/forum/#!msg/mechanical-sympathy/icNZJejUHfE/BfDekfBEs_sJ
2. https://github.com/HdrHistogram/HdrHistogram
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (c) 2012 - 2016 YCSB contributors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You
may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License. See accompanying
LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>site.ycsb</groupId>
<artifactId>root</artifactId>
<version>0.18.0-SNAPSHOT</version>
</parent>
<artifactId>core</artifactId>
<name>Core YCSB</name>
<packaging>jar</packaging>
<properties>
<jackson.api.version>1.9.4</jackson.api.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core4</artifactId>
<version>4.1.0-incubating</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
<version>${jackson.api.version}</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
<version>${jackson.api.version}</version>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>6.1.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hdrhistogram</groupId>
<artifactId>HdrHistogram</artifactId>
<version>2.1.4</version>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
</build>
<profiles>
<profile>
<!-- Build profile when running via yscb.sh or yscb.bat-->
<id>source-run</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>stage-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
/**
* Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package site.ycsb;
import java.util.*;
import java.util.Map.Entry;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.LockSupport;
/**
* Basic DB that just prints out the requested operations, instead of doing them against a database.
*/
public class BasicDB extends DB {
public static final String COUNT = "basicdb.count";
public static final String COUNT_DEFAULT = "false";
public static final String VERBOSE = "basicdb.verbose";
public static final String VERBOSE_DEFAULT = "true";
public static final String SIMULATE_DELAY = "basicdb.simulatedelay";
public static final String SIMULATE_DELAY_DEFAULT = "0";
public static final String RANDOMIZE_DELAY = "basicdb.randomizedelay";
public static final String RANDOMIZE_DELAY_DEFAULT = "true";
protected static final Object MUTEX = new Object();
protected static int counter = 0;
protected static Map<Integer, Integer> reads;
protected static Map<Integer, Integer> scans;
protected static Map<Integer, Integer> updates;
protected static Map<Integer, Integer> inserts;
protected static Map<Integer, Integer> deletes;
protected boolean verbose;
protected boolean randomizedelay;
protected int todelay;
protected boolean count;
public BasicDB() {
todelay = 0;
}
protected void delay() {
if (todelay > 0) {
long delayNs;
if (randomizedelay) {
delayNs = TimeUnit.MILLISECONDS.toNanos(ThreadLocalRandom.current().nextInt(todelay));
if (delayNs == 0) {
return;
}
} else {
delayNs = TimeUnit.MILLISECONDS.toNanos(todelay);
}
final long deadline = System.nanoTime() + delayNs;
do {
LockSupport.parkNanos(deadline - System.nanoTime());
} while (System.nanoTime() < deadline && !Thread.interrupted());
}
}
/**
* Initialize any state for this DB.
* Called once per DB instance; there is one DB instance per client thread.
*/
public void init() {
verbose = Boolean.parseBoolean(getProperties().getProperty(VERBOSE, VERBOSE_DEFAULT));
todelay = Integer.parseInt(getProperties().getProperty(SIMULATE_DELAY, SIMULATE_DELAY_DEFAULT));
randomizedelay = Boolean.parseBoolean(getProperties().getProperty(RANDOMIZE_DELAY, RANDOMIZE_DELAY_DEFAULT));
count = Boolean.parseBoolean(getProperties().getProperty(COUNT, COUNT_DEFAULT));
if (verbose) {
synchronized (System.out) {
System.out.println("***************** properties *****************");
Properties p = getProperties();
if (p != null) {
for (Enumeration e = p.propertyNames(); e.hasMoreElements();) {
String k = (String) e.nextElement();
System.out.println("\"" + k + "\"=\"" + p.getProperty(k) + "\"");
}
}
System.out.println("**********************************************");
}
}
synchronized (MUTEX) {
if (counter == 0 && count) {
reads = new HashMap<Integer, Integer>();
scans = new HashMap<Integer, Integer>();
updates = new HashMap<Integer, Integer>();
inserts = new HashMap<Integer, Integer>();
deletes = new HashMap<Integer, Integer>();
}
counter++;
}
}
protected static final ThreadLocal<StringBuilder> TL_STRING_BUILDER = new ThreadLocal<StringBuilder>() {
@Override
protected StringBuilder initialValue() {
return new StringBuilder();
}
};
protected static StringBuilder getStringBuilder() {
StringBuilder sb = TL_STRING_BUILDER.get();
sb.setLength(0);
return sb;
}
/**
* Read a record from the database. Each field/value pair from the result will be stored in a HashMap.
*
* @param table The name of the table
* @param key The record key of the record to read.
* @param fields The list of fields to read, or null for all of them
* @param result A HashMap of field/value pairs for the result
* @return Zero on success, a non-zero error code on error
*/
public Status read(String table, String key, Set<String> fields, Map<String, ByteIterator> result) {
delay();
if (verbose) {
StringBuilder sb = getStringBuilder();
sb.append("READ ").append(table).append(" ").append(key).append(" [ ");
if (fields != null) {
for (String f : fields) {
sb.append(f).append(" ");
}
} else {
sb.append("<all fields>");
}
sb.append("]");
System.out.println(sb);
}
if (count) {
incCounter(reads, hash(table, key, fields));
}
return Status.OK;
}
/**
* Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored
* in a HashMap.
*
* @param table The name of the table
* @param startkey The record key of the first record to read.
* @param recordcount The number of records to read
* @param fields The list of fields to read, or null for all of them
* @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record
* @return Zero on success, a non-zero error code on error
*/
public Status scan(String table, String startkey, int recordcount, Set<String> fields,
Vector<HashMap<String, ByteIterator>> result) {
delay();
if (verbose) {
StringBuilder sb = getStringBuilder();
sb.append("SCAN ").append(table).append(" ").append(startkey).append(" ").append(recordcount).append(" [ ");
if (fields != null) {
for (String f : fields) {
sb.append(f).append(" ");
}
} else {
sb.append("<all fields>");
}
sb.append("]");
System.out.println(sb);
}
if (count) {
incCounter(scans, hash(table, startkey, fields));
}
return Status.OK;
}
/**
* Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the
* record with the specified record key, overwriting any existing values with the same field name.
*
* @param table The name of the table
* @param key The record key of the record to write.
* @param values A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error
*/
public Status update(String table, String key, Map<String, ByteIterator> values) {
delay();
if (verbose) {
StringBuilder sb = getStringBuilder();
sb.append("UPDATE ").append(table).append(" ").append(key).append(" [ ");
if (values != null) {
for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
sb.append(entry.getKey()).append("=").append(entry.getValue()).append(" ");
}
}
sb.append("]");
System.out.println(sb);
}
if (count) {
incCounter(updates, hash(table, key, values));
}
return Status.OK;
}
/**
* Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the
* record with the specified record key.
*
* @param table The name of the table
* @param key The record key of the record to insert.
* @param values A HashMap of field/value pairs to insert in the record
* @return Zero on success, a non-zero error code on error
*/
public Status insert(String table, String key, Map<String, ByteIterator> values) {
delay();
if (verbose) {
StringBuilder sb = getStringBuilder();
sb.append("INSERT ").append(table).append(" ").append(key).append(" [ ");
if (values != null) {
for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
sb.append(entry.getKey()).append("=").append(entry.getValue()).append(" ");
}
}
sb.append("]");
System.out.println(sb);
}
if (count) {
incCounter(inserts, hash(table, key, values));
}
return Status.OK;
}
/**
* Delete a record from the database.
*
* @param table The name of the table
* @param key The record key of the record to delete.
* @return Zero on success, a non-zero error code on error
*/
public Status delete(String table, String key) {
delay();
if (verbose) {
StringBuilder sb = getStringBuilder();
sb.append("DELETE ").append(table).append(" ").append(key);
System.out.println(sb);
}
if (count) {
incCounter(deletes, (table + key).hashCode());
}
return Status.OK;
}
@Override
public void cleanup() {
synchronized (MUTEX) {
int countDown = --counter;
if (count && countDown < 1) {
// TODO - would be nice to call something like:
// Measurements.getMeasurements().oneOffMeasurement("READS", "Uniques", reads.size());
System.out.println("[READS], Uniques, " + reads.size());
System.out.println("[SCANS], Uniques, " + scans.size());
System.out.println("[UPDATES], Uniques, " + updates.size());
System.out.println("[INSERTS], Uniques, " + inserts.size());
System.out.println("[DELETES], Uniques, " + deletes.size());
}
}
}
/**
* Increments the count on the hash in the map.
* @param map A non-null map to sync and use for incrementing.
* @param hash A hash code to increment.
*/
protected void incCounter(final Map<Integer, Integer> map, final int hash) {
synchronized (map) {
Integer ctr = map.get(hash);
if (ctr == null) {
map.put(hash, 1);
} else {
map.put(hash, ctr + 1);
}
}
}
/**
* Hashes the table, key and fields, sorting the fields first for a consistent
* hash.
* Note that this is expensive as we generate a copy of the fields and a string
* buffer to hash on. Hashing on the objects is problematic.
* @param table The user table.
* @param key The key read or scanned.
* @param fields The fields read or scanned.
* @return The hash code.
*/
protected int hash(final String table, final String key, final Set<String> fields) {
if (fields == null) {
return (table + key).hashCode();
}
StringBuilder buf = getStringBuilder().append(table).append(key);
List<String> sorted = new ArrayList<String>(fields);
Collections.sort(sorted);
for (final String field : sorted) {
buf.append(field);
}
return buf.toString().hashCode();
}
/**
* Hashes the table, key and fields, sorting the fields first for a consistent
* hash.
* Note that this is expensive as we generate a copy of the fields and a string
* buffer to hash on. Hashing on the objects is problematic.
* @param table The user table.
* @param key The key read or scanned.
* @param values The values to hash on.
* @return The hash code.
*/
protected int hash(final String table, final String key, final Map<String, ByteIterator> values) {
if (values == null) {
return (table + key).hashCode();
}
final TreeMap<String, ByteIterator> sorted =
new TreeMap<String, ByteIterator>(values);
StringBuilder buf = getStringBuilder().append(table).append(key);
for (final Entry<String, ByteIterator> entry : sorted.entrySet()) {
entry.getValue().reset();
buf.append(entry.getKey())
.append(entry.getValue().toString());
}
return buf.toString().hashCode();
}
/**
* Short test of BasicDB
*/
/*
public static void main(String[] args) {
BasicDB bdb = new BasicDB();
Properties p = new Properties();
p.setProperty("Sky", "Blue");
p.setProperty("Ocean", "Wet");
bdb.setProperties(p);
bdb.init();
HashMap<String, String> fields = new HashMap<String, ByteIterator>();
fields.put("A", new StringByteIterator("X"));
fields.put("B", new StringByteIterator("Y"));
bdb.read("table", "key", null, null);
bdb.insert("table", "key", fields);
fields = new HashMap<String, ByteIterator>();
fields.put("C", new StringByteIterator("Z"));
bdb.update("table", "key", fields);
bdb.delete("table", "key");
}
*/
}
/**
* Copyright (c) 2017 YCSB contributors All rights reserved.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package site.ycsb;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import site.ycsb.workloads.TimeSeriesWorkload;
/**
* Basic DB for printing out time series workloads and/or tracking the distribution
* of keys and fields.
*/
public class BasicTSDB extends BasicDB {
/** Time series workload specific counters. */
protected static Map<Long, Integer> timestamps;
protected static Map<Integer, Integer> floats;
protected static Map<Integer, Integer> integers;
private String timestampKey;
private String valueKey;
private String tagPairDelimiter;
private String queryTimeSpanDelimiter;
private long lastTimestamp;
@Override
public void init() {
super.init();
synchronized (MUTEX) {
if (timestamps == null) {
timestamps = new HashMap<Long, Integer>();
floats = new HashMap<Integer, Integer>();
integers = new HashMap<Integer, Integer>();
}
}
timestampKey = getProperties().getProperty(
TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY,
TimeSeriesWorkload.TIMESTAMP_KEY_PROPERTY_DEFAULT);
valueKey = getProperties().getProperty(
TimeSeriesWorkload.VALUE_KEY_PROPERTY,
TimeSeriesWorkload.VALUE_KEY_PROPERTY_DEFAULT);
tagPairDelimiter = getProperties().getProperty(
TimeSeriesWorkload.PAIR_DELIMITER_PROPERTY,
TimeSeriesWorkload.PAIR_DELIMITER_PROPERTY_DEFAULT);
queryTimeSpanDelimiter = getProperties().getProperty(
TimeSeriesWorkload.QUERY_TIMESPAN_DELIMITER_PROPERTY,
TimeSeriesWorkload.QUERY_TIMESPAN_DELIMITER_PROPERTY_DEFAULT);
}
public Status read(String table, String key, Set<String> fields, Map<String, ByteIterator> result) {
delay();
if (verbose) {
StringBuilder sb = getStringBuilder();
sb.append("READ ").append(table).append(" ").append(key).append(" [ ");
if (fields != null) {
for (String f : fields) {
sb.append(f).append(" ");
}
} else {
sb.append("<all fields>");
}
sb.append("]");
System.out.println(sb);
}
if (count) {
Set<String> filtered = null;
if (fields != null) {
filtered = new HashSet<String>();
for (final String field : fields) {
if (field.startsWith(timestampKey)) {
String[] parts = field.split(tagPairDelimiter);
if (parts[1].contains(queryTimeSpanDelimiter)) {
parts = parts[1].split(queryTimeSpanDelimiter);
lastTimestamp = Long.parseLong(parts[0]);
} else {
lastTimestamp = Long.parseLong(parts[1]);
}
synchronized(timestamps) {
Integer ctr = timestamps.get(lastTimestamp);
if (ctr == null) {
timestamps.put(lastTimestamp, 1);
} else {
timestamps.put(lastTimestamp, ctr + 1);
}
}
} else {
filtered.add(field);
}
}
}
incCounter(reads, hash(table, key, filtered));
}
return Status.OK;
}
@Override
public Status update(String table, String key, Map<String, ByteIterator> values) {
delay();
boolean isFloat = false;
if (verbose) {
StringBuilder sb = getStringBuilder();
sb.append("UPDATE ").append(table).append(" ").append(key).append(" [ ");
if (values != null) {
final TreeMap<String, ByteIterator> tree = new TreeMap<String, ByteIterator>(values);
for (Map.Entry<String, ByteIterator> entry : tree.entrySet()) {
if (entry.getKey().equals(timestampKey)) {
sb.append(entry.getKey()).append("=")
.append(Utils.bytesToLong(entry.getValue().toArray())).append(" ");
} else if (entry.getKey().equals(valueKey)) {
final NumericByteIterator it = (NumericByteIterator) entry.getValue();
isFloat = it.isFloatingPoint();
sb.append(entry.getKey()).append("=")
.append(isFloat ? it.getDouble() : it.getLong()).append(" ");
} else {
sb.append(entry.getKey()).append("=").append(entry.getValue()).append(" ");
}
}
}
sb.append("]");
System.out.println(sb);
}
if (count) {
if (!verbose) {
isFloat = ((NumericByteIterator) values.get(valueKey)).isFloatingPoint();
}
int hash = hash(table, key, values);
incCounter(updates, hash);
synchronized(timestamps) {
Integer ctr = timestamps.get(lastTimestamp);
if (ctr == null) {
timestamps.put(lastTimestamp, 1);
} else {
timestamps.put(lastTimestamp, ctr + 1);
}
}
if (isFloat) {
incCounter(floats, hash);
} else {
incCounter(integers, hash);
}
}
return Status.OK;
}
@Override
public Status insert(String table, String key, Map<String, ByteIterator> values) {
delay();
boolean isFloat = false;
if (verbose) {
StringBuilder sb = getStringBuilder();
sb.append("INSERT ").append(table).append(" ").append(key).append(" [ ");
if (values != null) {
final TreeMap<String, ByteIterator> tree = new TreeMap<String, ByteIterator>(values);
for (Map.Entry<String, ByteIterator> entry : tree.entrySet()) {
if (entry.getKey().equals(timestampKey)) {
sb.append(entry.getKey()).append("=")
.append(Utils.bytesToLong(entry.getValue().toArray())).append(" ");
} else if (entry.getKey().equals(valueKey)) {
final NumericByteIterator it = (NumericByteIterator) entry.getValue();
isFloat = it.isFloatingPoint();
sb.append(entry.getKey()).append("=")
.append(isFloat ? it.getDouble() : it.getLong()).append(" ");
} else {
sb.append(entry.getKey()).append("=").append(entry.getValue()).append(" ");
}
}
}
sb.append("]");
System.out.println(sb);
}
if (count) {
if (!verbose) {
isFloat = ((NumericByteIterator) values.get(valueKey)).isFloatingPoint();
}
int hash = hash(table, key, values);
incCounter(inserts, hash);
synchronized(timestamps) {
Integer ctr = timestamps.get(lastTimestamp);
if (ctr == null) {
timestamps.put(lastTimestamp, 1);
} else {
timestamps.put(lastTimestamp, ctr + 1);
}
}
if (isFloat) {
incCounter(floats, hash);
} else {
incCounter(integers, hash);
}
}
return Status.OK;
}
@Override
public void cleanup() {
super.cleanup();
if (count && counter < 1) {
System.out.println("[TIMESTAMPS], Unique, " + timestamps.size());
System.out.println("[FLOATS], Unique series, " + floats.size());
System.out.println("[INTEGERS], Unique series, " + integers.size());
long minTs = Long.MAX_VALUE;
long maxTs = Long.MIN_VALUE;
for (final long ts : timestamps.keySet()) {
if (ts > maxTs) {
maxTs = ts;
}
if (ts < minTs) {
minTs = ts;
}
}
System.out.println("[TIMESTAMPS], Min, " + minTs);
System.out.println("[TIMESTAMPS], Max, " + maxTs);
}
}
@Override
protected int hash(final String table, final String key, final Map<String, ByteIterator> values) {
final TreeMap<String, ByteIterator> sorted = new TreeMap<String, ByteIterator>();
for (final Entry<String, ByteIterator> entry : values.entrySet()) {
if (entry.getKey().equals(valueKey)) {
continue;
} else if (entry.getKey().equals(timestampKey)) {
lastTimestamp = ((NumericByteIterator) entry.getValue()).getLong();
entry.getValue().reset();
continue;
}
sorted.put(entry.getKey(), entry.getValue());
}
// yeah it's ugly but gives us a unique hash without having to add hashers
// to all of the ByteIterators.
StringBuilder buf = new StringBuilder().append(table).append(key);
for (final Entry<String, ByteIterator> entry : sorted.entrySet()) {
entry.getValue().reset();
buf.append(entry.getKey())
.append(entry.getValue().toString());
}
return buf.toString().hashCode();
}
}
\ No newline at end of file
/**
* Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package site.ycsb;
/**
* A ByteIterator that iterates through a byte array.
*/
public class ByteArrayByteIterator extends ByteIterator {
private final int originalOffset;
private final byte[] str;
private int off;
private final int len;
public ByteArrayByteIterator(byte[] s) {
this.str = s;
this.off = 0;
this.len = s.length;
originalOffset = 0;
}
public ByteArrayByteIterator(byte[] s, int off, int len) {
this.str = s;
this.off = off;
this.len = off + len;
originalOffset = off;
}
@Override
public boolean hasNext() {
return off < len;
}
@Override
public byte nextByte() {
byte ret = str[off];
off++;
return ret;
}
@Override
public long bytesLeft() {
return len - off;
}
@Override
public void reset() {
off = originalOffset;
}
@Override
public byte[] toArray() {
int size = (int) bytesLeft();
byte[] bytes = new byte[size];
System.arraycopy(str, off, bytes, 0, size);
off = len;
return bytes;
}
}
/**
* Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package site.ycsb;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.util.Iterator;
/**
* YCSB-specific buffer class. ByteIterators are designed to support
* efficient field generation, and to allow backend drivers that can stream
* fields (instead of materializing them in RAM) to do so.
* <p>
* YCSB originially used String objects to represent field values. This led to
* two performance issues.
* </p><p>
* First, it leads to unnecessary conversions between UTF-16 and UTF-8, both
* during field generation, and when passing data to byte-based backend
* drivers.
* </p><p>
* Second, Java strings are represented internally using UTF-16, and are
* built by appending to a growable array type (StringBuilder or
* StringBuffer), then calling a toString() method. This leads to a 4x memory
* overhead as field values are being built, which prevented YCSB from
* driving large object stores.
* </p>
* The StringByteIterator class contains a number of convenience methods for
* backend drivers that convert between Map&lt;String,String&gt; and
* Map&lt;String,ByteBuffer&gt;.
*
*/
public abstract class ByteIterator implements Iterator<Byte> {
@Override
public abstract boolean hasNext();
@Override
public Byte next() {
throw new UnsupportedOperationException();
}
public abstract byte nextByte();
/** @return byte offset immediately after the last valid byte */
public int nextBuf(byte[] buf, int bufOff) {
int sz = bufOff;
while (sz < buf.length && hasNext()) {
buf[sz] = nextByte();
sz++;
}
return sz;
}
public abstract long bytesLeft();
@Override
public void remove() {
throw new UnsupportedOperationException();
}
/** Resets the iterator so that it can be consumed again. Not all
* implementations support this call.
* @throws UnsupportedOperationException if the implementation hasn't implemented
* the method.
*/
public void reset() {
throw new UnsupportedOperationException();
}
/** Consumes remaining contents of this object, and returns them as a string. */
public String toString() {
Charset cset = Charset.forName("UTF-8");
CharBuffer cb = cset.decode(ByteBuffer.wrap(this.toArray()));
return cb.toString();
}
/** Consumes remaining contents of this object, and returns them as a byte array. */
public byte[] toArray() {
long left = bytesLeft();
if (left != (int) left) {
throw new ArrayIndexOutOfBoundsException("Too much data to fit in one array!");
}
byte[] ret = new byte[(int) left];
for (int i = 0; i < ret.length; i++) {
ret[i] = nextByte();
}
return ret;
}
}
/**
* Copyright (c) 2010-2016 Yahoo! Inc., 2017 YCSB contributors All rights reserved.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you
* may not use this file except in compliance with the License. You
* may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
* implied. See the License for the specific language governing
* permissions and limitations under the License. See accompanying
* LICENSE file.
*/
package site.ycsb;
import site.ycsb.measurements.Measurements;
import site.ycsb.measurements.exporter.MeasurementsExporter;
import site.ycsb.measurements.exporter.TextMeasurementsExporter;
import org.apache.htrace.core.HTraceConfiguration;
import org.apache.htrace.core.TraceScope;
import org.apache.htrace.core.Tracer;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
/**
* Turn seconds remaining into more useful units.
* i.e. if there are hours or days worth of seconds, use them.
*/
final class RemainingFormatter {
private RemainingFormatter() {
// not used
}
public static StringBuilder format(long seconds) {
StringBuilder time = new StringBuilder();
long days = TimeUnit.SECONDS.toDays(seconds);
if (days > 0) {
time.append(days).append(days == 1 ? " day " : " days ");
seconds -= TimeUnit.DAYS.toSeconds(days);
}
long hours = TimeUnit.SECONDS.toHours(seconds);
if (hours > 0) {
time.append(hours).append(hours == 1 ? " hour " : " hours ");
seconds -= TimeUnit.HOURS.toSeconds(hours);
}
/* Only include minute granularity if we're < 1 day. */
if (days < 1) {
long minutes = TimeUnit.SECONDS.toMinutes(seconds);
if (minutes > 0) {
time.append(minutes).append(minutes == 1 ? " minute " : " minutes ");
seconds -= TimeUnit.MINUTES.toSeconds(seconds);
}
}
/* Only bother to include seconds if we're < 1 minute */
if (time.length() == 0) {
time.append(seconds).append(time.length() == 1 ? " second " : " seconds ");
}
return time;
}
}
/**
* Main class for executing YCSB.
*/
public final class Client {
private Client() {
//not used
}
public static final String DEFAULT_RECORD_COUNT = "0";
/**
* The target number of operations to perform.
*/
public static final String OPERATION_COUNT_PROPERTY = "operationcount";
/**
* The number of records to load into the database initially.
*/
public static final String RECORD_COUNT_PROPERTY = "recordcount";
/**
* The workload class to be loaded.
*/
public static final String WORKLOAD_PROPERTY = "workload";
/**
* The database class to be used.
*/
public static final String DB_PROPERTY = "db";
/**
* The exporter class to be used. The default is
* site.ycsb.measurements.exporter.TextMeasurementsExporter.
*/
public static final String EXPORTER_PROPERTY = "exporter";
/**
* If set to the path of a file, YCSB will write all output to this file
* instead of STDOUT.
*/
public static final String EXPORT_FILE_PROPERTY = "exportfile";
/**
* The number of YCSB client threads to run.
*/
public static final String THREAD_COUNT_PROPERTY = "threadcount";
/**
* Indicates how many inserts to do if less than recordcount.
* Useful for partitioning the load among multiple servers if the client is the bottleneck.
* Additionally workloads should support the "insertstart" property which tells them which record to start at.
*/
public static final String INSERT_COUNT_PROPERTY = "insertcount";
/**
* Target number of operations per second.
*/
public static final String TARGET_PROPERTY = "target";
/**
* The maximum amount of time (in seconds) for which the benchmark will be run.
*/
public static final String MAX_EXECUTION_TIME = "maxexecutiontime";
/**
* Whether or not this is the transaction phase (run) or not (load).
*/
public static final String DO_TRANSACTIONS_PROPERTY = "dotransactions";
/**
* Whether or not to show status during run.
*/
public static final String STATUS_PROPERTY = "status";
/**
* Use label for status (e.g. to label one experiment out of a whole batch).
*/
public static final String LABEL_PROPERTY = "label";
/**
* An optional thread used to track progress and measure JVM stats.
*/
private static StatusThread statusthread = null;
// HTrace integration related constants.
/**
* All keys for configuring the tracing system start with this prefix.
*/
private static final String HTRACE_KEY_PREFIX = "htrace.";
private static final String CLIENT_WORKLOAD_INIT_SPAN = "Client#workload_init";
private static final String CLIENT_INIT_SPAN = "Client#init";
private static final String CLIENT_WORKLOAD_SPAN = "Client#workload";
private static final String CLIENT_CLEANUP_SPAN = "Client#cleanup";
private static final String CLIENT_EXPORT_MEASUREMENTS_SPAN = "Client#export_measurements";
public static void usageMessage() {
System.out.println("Usage: java site.ycsb.Client [options]");
System.out.println("Options:");
System.out.println(" -threads n: execute using n threads (default: 1) - can also be specified as the \n" +
" \"threadcount\" property using -p");
System.out.println(" -target n: attempt to do n operations per second (default: unlimited) - can also\n" +
" be specified as the \"target\" property using -p");
System.out.println(" -load: run the loading phase of the workload");
System.out.println(" -t: run the transactions phase of the workload (default)");
System.out.println(" -db dbname: specify the name of the DB to use (default: site.ycsb.BasicDB) - \n" +
" can also be specified as the \"db\" property using -p");
System.out.println(" -P propertyfile: load properties from the given file. Multiple files can");
System.out.println(" be specified, and will be processed in the order specified");
System.out.println(" -p name=value: specify a property to be passed to the DB and workloads;");
System.out.println(" multiple properties can be specified, and override any");
System.out.println(" values in the propertyfile");
System.out.println(" -s: show status during run (default: no status)");
System.out.println(" -l label: use label for status (e.g. to label one experiment out of a whole batch)");
System.out.println("");
System.out.println("Required properties:");
System.out.println(" " + WORKLOAD_PROPERTY + ": the name of the workload class to use (e.g. " +
"site.ycsb.workloads.CoreWorkload)");
System.out.println("");
System.out.println("To run the transaction phase from multiple servers, start a separate client on each.");
System.out.println("To run the load phase from multiple servers, start a separate client on each; additionally,");
System.out.println("use the \"insertcount\" and \"insertstart\" properties to divide up the records " +
"to be inserted");
}
public static boolean checkRequiredProperties(Properties props) {
if (props.getProperty(WORKLOAD_PROPERTY) == null) {
System.out.println("Missing property: " + WORKLOAD_PROPERTY);
return false;
}
return true;
}
/**
* Exports the measurements to either sysout or a file using the exporter
* loaded from conf.
*
* @throws IOException Either failed to write to output stream or failed to close it.
*/
private static void exportMeasurements(Properties props, int opcount, long runtime)
throws IOException {
MeasurementsExporter exporter = null;
try {
// if no destination file is provided the results will be written to stdout
OutputStream out;
String exportFile = props.getProperty(EXPORT_FILE_PROPERTY);
if (exportFile == null) {
out = System.out;
} else {
out = new FileOutputStream(exportFile);
}
// if no exporter is provided the default text one will be used
String exporterStr = props.getProperty(EXPORTER_PROPERTY,
"site.ycsb.measurements.exporter.TextMeasurementsExporter");
try {
exporter = (MeasurementsExporter) Class.forName(exporterStr).getConstructor(OutputStream.class)
.newInstance(out);
} catch (Exception e) {
System.err.println("Could not find exporter " + exporterStr
+ ", will use default text reporter.");
e.printStackTrace();
exporter = new TextMeasurementsExporter(out);
}
exporter.write("OVERALL", "RunTime(ms)", runtime);
double throughput = 1000.0 * (opcount) / (runtime);
exporter.write("OVERALL", "Throughput(ops/sec)", throughput);
final Map<String, Long[]> gcs = Utils.getGCStatst();
long totalGCCount = 0;
long totalGCTime = 0;
for (final Entry<String, Long[]> entry : gcs.entrySet()) {
exporter.write("TOTAL_GCS_" + entry.getKey(), "Count", entry.getValue()[0]);
exporter.write("TOTAL_GC_TIME_" + entry.getKey(), "Time(ms)", entry.getValue()[1]);
exporter.write("TOTAL_GC_TIME_%_" + entry.getKey(), "Time(%)",
((double) entry.getValue()[1] /